feat(tower): shared content module
This commit is contained in:
Clément Saillant
@@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared content logic for the Lisael Box on Tower: feeds config, RSS, transcode,
|
||||
covers, staging/manifest ops, box state, push. Imported by lisael_refresh.py (cron)
|
||||
and lisael_server.py (web)."""
|
||||
import hashlib, io, json, os, ssl, subprocess, sys, tempfile, threading, urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from PIL import Image
|
||||
|
||||
BASE = os.environ.get("LISAEL_BASE", "/home/clems/lisael-content")
|
||||
STAGING = os.environ.get("LISAEL_STAGING", os.path.join(BASE, "podcasts"))
|
||||
LVGLIMG = os.environ.get("LISAEL_LVGLIMG", os.path.join(BASE, "LVGLImage.py"))
|
||||
FEEDS_JSON = os.path.join(BASE, "feeds.json")
|
||||
BOX_JSON = os.path.join(BASE, "box-state.json")
|
||||
BOX_PORT = int(os.environ.get("LISAEL_BOX_PORT", "8080"))
|
||||
UA = {"User-Agent": "Mozilla/5.0"}
|
||||
NS = {"itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd"}
|
||||
SUBS = {"’": "'", "“": '"', "”": '"', "–": "-",
|
||||
"—": "-", "…": "...", "«": '"', "»": '"'}
|
||||
|
||||
DEFAULT_FEEDS = [
|
||||
{"key": "oli", "name": "Une histoire et Oli", "rss": "https://radiofrance-podcast.net/podcast09/rss_19721.xml", "n": 3},
|
||||
{"key": "odyssees", "name": "Les Odyssees", "rss": "https://radiofrance-podcast.net/podcast09/rss_20108.xml", "n": 3},
|
||||
{"key": "salutinfo", "name": "Salut l'info", "rss": "https://radiofrance-podcast.net/podcast09/rss_20689.xml", "n": 2},
|
||||
{"key": "bestioles", "name": "Bestioles", "rss": "https://radiofrance-podcast.net/podcast09/35099478-7c72-4f9e-a6de-1b928400e9e5/podcast_a80ecbd5-df3d-4c9d-bee7-4e3d9efc1974.xml", "n": 2},
|
||||
]
|
||||
|
||||
def fetch(u, t=180):
|
||||
return urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=t,
|
||||
context=ssl.create_default_context()).read()
|
||||
|
||||
def clean(s):
|
||||
for a, b in SUBS.items():
|
||||
s = s.replace(a, b)
|
||||
return " ".join(s.split()).strip()
|
||||
|
||||
def _atomic_write_bytes(path, data):
|
||||
d = os.path.dirname(path) or "."
|
||||
os.makedirs(d, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=d)
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(data)
|
||||
os.replace(tmp, path)
|
||||
|
||||
def _atomic_write_json(path, obj):
|
||||
_atomic_write_bytes(path, json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8"))
|
||||
|
||||
# ---- feeds config ----
|
||||
def load_feeds():
|
||||
if os.path.exists(FEEDS_JSON):
|
||||
try:
|
||||
return json.load(open(FEEDS_JSON, encoding="utf-8"))
|
||||
except Exception:
|
||||
pass
|
||||
save_feeds(DEFAULT_FEEDS)
|
||||
return [dict(f) for f in DEFAULT_FEEDS]
|
||||
|
||||
def save_feeds(feeds):
|
||||
_atomic_write_json(FEEDS_JSON, feeds)
|
||||
|
||||
def add_feed(key, name, rss, n=2):
|
||||
feeds = load_feeds()
|
||||
feeds = [f for f in feeds if f["key"] != key]
|
||||
feeds.append({"key": key, "name": name, "rss": rss, "n": int(n)})
|
||||
save_feeds(feeds)
|
||||
return feeds
|
||||
|
||||
def remove_feed(key):
|
||||
save_feeds([f for f in load_feeds() if f["key"] != key])
|
||||
|
||||
# ---- RSS ----
|
||||
def cover_url(root):
|
||||
el = root.find(".//channel/itunes:image", NS)
|
||||
if el is not None and el.get("href"):
|
||||
return el.get("href")
|
||||
el = root.find(".//channel/image/url")
|
||||
return el.text if el is not None else None
|
||||
|
||||
def fetch_episodes(rss, limit=None):
|
||||
"""Return ([{url,title,duration,guid}], cover_href)."""
|
||||
root = ET.fromstring(fetch(rss))
|
||||
items = root.findall(".//item")
|
||||
if limit:
|
||||
items = items[:limit]
|
||||
out = []
|
||||
for it in items:
|
||||
enc = it.find("enclosure")
|
||||
if enc is None or not enc.get("url"):
|
||||
continue
|
||||
out.append({
|
||||
"url": enc.get("url"),
|
||||
"title": clean(it.findtext("title") or "Episode"),
|
||||
"duration": (it.findtext("itunes:duration", namespaces=NS) or "").strip(),
|
||||
"guid": (it.findtext("guid") or enc.get("url")),
|
||||
})
|
||||
return out, cover_url(root)
|
||||
|
||||
# ---- transcode / cover ----
|
||||
def episode_fname(key, url):
|
||||
return f"{key}_{hashlib.sha1(url.encode()).hexdigest()[:8]}_a48.mp3"
|
||||
|
||||
def transcode(raw_path, out_path):
|
||||
rc = subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", raw_path,
|
||||
"-ac", "1", "-b:a", "48k", "-c:a", "libmp3lame", out_path],
|
||||
capture_output=True).returncode
|
||||
return rc == 0 and os.path.exists(out_path)
|
||||
|
||||
def make_cover(img_bytes, key):
|
||||
"""160x160 RGB565 .bin into STAGING via LVGLImage. Returns '<key>.bin' or ''."""
|
||||
try:
|
||||
im = Image.open(io.BytesIO(img_bytes)).convert("RGB").resize((160, 160))
|
||||
os.makedirs(STAGING, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
png = os.path.join(td, f"{key}.png"); im.save(png)
|
||||
subprocess.run([sys.executable, LVGLIMG, "--ofmt", "BIN", "--cf", "RGB565",
|
||||
"-o", td, png], check=True, capture_output=True)
|
||||
binp = os.path.join(td, f"{key}.bin")
|
||||
if os.path.exists(binp):
|
||||
os.replace(binp, os.path.join(STAGING, f"{key}.bin"))
|
||||
return f"{key}.bin"
|
||||
except Exception as e:
|
||||
print("cover fail", key, e, flush=True)
|
||||
return ""
|
||||
|
||||
# ---- staging / manifest ----
|
||||
def staging_files():
|
||||
if not os.path.isdir(STAGING):
|
||||
return []
|
||||
return sorted(f for f in os.listdir(STAGING) if not f.startswith("."))
|
||||
|
||||
def manifest_load():
|
||||
p = os.path.join(STAGING, "manifest.json")
|
||||
if os.path.exists(p):
|
||||
try:
|
||||
return json.load(open(p, encoding="utf-8"))
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
def manifest_save(m):
|
||||
_atomic_write_json(os.path.join(STAGING, "manifest.json"), m)
|
||||
|
||||
def _manifest_upsert(m, show, cover, file, title):
|
||||
for s in m:
|
||||
if s["show"] == show:
|
||||
if cover and not s.get("cover"):
|
||||
s["cover"] = cover
|
||||
for e in s["episodes"]:
|
||||
if e["file"] == file:
|
||||
e["title"] = title
|
||||
return
|
||||
s["episodes"].append({"file": file, "title": title})
|
||||
return
|
||||
m.append({"show": show, "cover": cover or "", "episodes": [{"file": file, "title": title}]})
|
||||
|
||||
def add_episode(key, show, cover_bytes, url, title):
|
||||
"""Download + transcode an episode into STAGING + upsert manifest. Idempotent by fname.
|
||||
cover_bytes: raw image bytes for the show cover, or None."""
|
||||
os.makedirs(STAGING, exist_ok=True)
|
||||
fname = episode_fname(key, url)
|
||||
out = os.path.join(STAGING, fname)
|
||||
if not os.path.exists(out):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
raw = os.path.join(td, "raw"); open(raw, "wb").write(fetch(url))
|
||||
tmp_out = os.path.join(td, fname)
|
||||
if not transcode(raw, tmp_out):
|
||||
raise RuntimeError(f"transcode failed {fname}")
|
||||
os.replace(tmp_out, out)
|
||||
cover = make_cover(cover_bytes, key) if cover_bytes else ""
|
||||
m = manifest_load()
|
||||
_manifest_upsert(m, show, cover, fname, clean(title))
|
||||
manifest_save(m)
|
||||
return fname
|
||||
|
||||
def add_upload(show, src_bytes, title, key="perso"):
|
||||
os.makedirs(STAGING, exist_ok=True)
|
||||
h = hashlib.sha1(src_bytes[:65536] + str(len(src_bytes)).encode()).hexdigest()[:8]
|
||||
fname = f"{key}_{h}_a48.mp3"
|
||||
out = os.path.join(STAGING, fname)
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
raw = os.path.join(td, "raw"); open(raw, "wb").write(src_bytes)
|
||||
tmp_out = os.path.join(td, fname)
|
||||
if not transcode(raw, tmp_out):
|
||||
raise RuntimeError("transcode failed")
|
||||
os.replace(tmp_out, out)
|
||||
m = manifest_load()
|
||||
_manifest_upsert(m, show, "", fname, clean(title))
|
||||
manifest_save(m)
|
||||
return fname
|
||||
|
||||
def remove_file(name):
|
||||
if "/" in name or ".." in name or name.startswith("."):
|
||||
raise ValueError("bad name")
|
||||
p = os.path.join(STAGING, name)
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
m = manifest_load()
|
||||
for s in m:
|
||||
s["episodes"] = [e for e in s["episodes"] if e["file"] != name]
|
||||
m = [s for s in m if s["episodes"]]
|
||||
manifest_save(m)
|
||||
|
||||
# ---- box state + push ----
|
||||
def box_state_load():
|
||||
if os.path.exists(BOX_JSON):
|
||||
try:
|
||||
return json.load(open(BOX_JSON, encoding="utf-8"))
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
def box_state_save(st):
|
||||
_atomic_write_json(BOX_JSON, st)
|
||||
|
||||
def compute_delta(staging, have):
|
||||
haveset = set(have)
|
||||
files = [f for f in staging if f != "manifest.json" and f not in haveset]
|
||||
if files and "manifest.json" in staging:
|
||||
files.append("manifest.json")
|
||||
return files
|
||||
|
||||
def push_file(ip, name):
|
||||
data = open(os.path.join(STAGING, name), "rb").read()
|
||||
url = f"http://{ip}:{BOX_PORT}/put?name=podcasts/{name}"
|
||||
with urllib.request.urlopen(urllib.request.Request(url, data=data, method="POST"),
|
||||
timeout=300) as r:
|
||||
return r.status
|
||||
|
||||
_busy = set()
|
||||
_busy_lock = threading.Lock()
|
||||
|
||||
def push_batch(ip, delta):
|
||||
ok = 0
|
||||
for f in delta:
|
||||
try:
|
||||
push_file(ip, f); ok += 1
|
||||
except Exception as e:
|
||||
print("push fail", f, e, flush=True)
|
||||
print(f"push done {ip} {ok}/{len(delta)}", flush=True)
|
||||
with _busy_lock:
|
||||
_busy.discard(ip)
|
||||
|
||||
def try_push(ip, delta):
|
||||
"""Start a background push unless one already runs for ip. Returns started?"""
|
||||
if not delta:
|
||||
return False
|
||||
with _busy_lock:
|
||||
if ip in _busy:
|
||||
return False
|
||||
_busy.add(ip)
|
||||
threading.Thread(target=push_batch, args=(ip, delta), daemon=True).start()
|
||||
return True
|
||||
@@ -0,0 +1,48 @@
|
||||
import os, tempfile, json
|
||||
os.environ["LISAEL_BASE"] = tempfile.mkdtemp(prefix="merlin_test_")
|
||||
os.environ["LISAEL_STAGING"] = os.path.join(os.environ["LISAEL_BASE"], "podcasts")
|
||||
import lisael_content as C
|
||||
|
||||
def test_episode_fname_stable():
|
||||
a = C.episode_fname("oli", "http://x/ep1.m4a")
|
||||
assert a == C.episode_fname("oli", "http://x/ep1.m4a")
|
||||
assert a.endswith("_a48.mp3") and a.startswith("oli_")
|
||||
assert a != C.episode_fname("oli", "http://x/ep2.m4a")
|
||||
|
||||
def test_feeds_default_seed():
|
||||
feeds = C.load_feeds()
|
||||
assert any(f["key"] == "oli" for f in feeds)
|
||||
assert os.path.exists(C.FEEDS_JSON)
|
||||
C.add_feed("zz", "Test", "http://r/ss", 1)
|
||||
assert any(f["key"] == "zz" for f in C.load_feeds())
|
||||
C.remove_feed("zz")
|
||||
assert not any(f["key"] == "zz" for f in C.load_feeds())
|
||||
|
||||
def test_manifest_upsert_remove():
|
||||
os.makedirs(C.STAGING, exist_ok=True)
|
||||
m = []
|
||||
C._manifest_upsert(m, "Oli", "oli.bin", "oli_a_a48.mp3", "Ep A")
|
||||
C._manifest_upsert(m, "Oli", "", "oli_b_a48.mp3", "Ep B")
|
||||
assert len(m) == 1 and len(m[0]["episodes"]) == 2 and m[0]["cover"] == "oli.bin"
|
||||
C.manifest_save(m)
|
||||
open(os.path.join(C.STAGING, "oli_a_a48.mp3"), "wb").close()
|
||||
C.remove_file("oli_a_a48.mp3")
|
||||
m2 = C.manifest_load()
|
||||
assert [e["file"] for e in m2[0]["episodes"]] == ["oli_b_a48.mp3"]
|
||||
|
||||
def test_compute_delta():
|
||||
assert C.compute_delta([], []) == []
|
||||
assert set(C.compute_delta(["a.mp3", "manifest.json"], [])) == {"a.mp3", "manifest.json"}
|
||||
assert C.compute_delta(["a.mp3", "manifest.json"], ["a.mp3"]) == []
|
||||
|
||||
def test_bad_name_rejected():
|
||||
try:
|
||||
C.remove_file("../x"); assert False
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
for name, fn in sorted(globals().items()):
|
||||
if name.startswith("test_"):
|
||||
fn(); print("ok", name)
|
||||
print("ALL OK")
|
||||
Reference in New Issue
Block a user