From 0ca9df54962a02edc1ce27daf635786e4ec22629 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Saillant?= Date: Mon, 15 Jun 2026 07:59:14 +0200 Subject: [PATCH] feat(tower): shared content module --- tower/lisael_content.py | 251 ++++++++++++++++++++++++++++++++++++++++ tower/test_content.py | 48 ++++++++ 2 files changed, 299 insertions(+) create mode 100644 tower/lisael_content.py create mode 100644 tower/test_content.py diff --git a/tower/lisael_content.py b/tower/lisael_content.py new file mode 100644 index 0000000..953f4ea --- /dev/null +++ b/tower/lisael_content.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Shared content logic for the Lisael Box on Tower: feeds config, RSS, transcode, +covers, staging/manifest ops, box state, push. Imported by lisael_refresh.py (cron) +and lisael_server.py (web).""" +import hashlib, io, json, os, ssl, subprocess, sys, tempfile, threading, urllib.request +import xml.etree.ElementTree as ET +from PIL import Image + +BASE = os.environ.get("LISAEL_BASE", "/home/clems/lisael-content") +STAGING = os.environ.get("LISAEL_STAGING", os.path.join(BASE, "podcasts")) +LVGLIMG = os.environ.get("LISAEL_LVGLIMG", os.path.join(BASE, "LVGLImage.py")) +FEEDS_JSON = os.path.join(BASE, "feeds.json") +BOX_JSON = os.path.join(BASE, "box-state.json") +BOX_PORT = int(os.environ.get("LISAEL_BOX_PORT", "8080")) +UA = {"User-Agent": "Mozilla/5.0"} +NS = {"itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd"} +SUBS = {"’": "'", "“": '"', "”": '"', "–": "-", + "—": "-", "…": "...", "«": '"', "»": '"'} + +DEFAULT_FEEDS = [ + {"key": "oli", "name": "Une histoire et Oli", "rss": "https://radiofrance-podcast.net/podcast09/rss_19721.xml", "n": 3}, + {"key": "odyssees", "name": "Les Odyssees", "rss": "https://radiofrance-podcast.net/podcast09/rss_20108.xml", "n": 3}, + {"key": "salutinfo", "name": "Salut l'info", "rss": "https://radiofrance-podcast.net/podcast09/rss_20689.xml", "n": 2}, + {"key": "bestioles", "name": "Bestioles", "rss": "https://radiofrance-podcast.net/podcast09/35099478-7c72-4f9e-a6de-1b928400e9e5/podcast_a80ecbd5-df3d-4c9d-bee7-4e3d9efc1974.xml", "n": 2}, +] + +def fetch(u, t=180): + return urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=t, + context=ssl.create_default_context()).read() + +def clean(s): + for a, b in SUBS.items(): + s = s.replace(a, b) + return " ".join(s.split()).strip() + +def _atomic_write_bytes(path, data): + d = os.path.dirname(path) or "." + os.makedirs(d, exist_ok=True) + fd, tmp = tempfile.mkstemp(dir=d) + with os.fdopen(fd, "wb") as f: + f.write(data) + os.replace(tmp, path) + +def _atomic_write_json(path, obj): + _atomic_write_bytes(path, json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8")) + +# ---- feeds config ---- +def load_feeds(): + if os.path.exists(FEEDS_JSON): + try: + return json.load(open(FEEDS_JSON, encoding="utf-8")) + except Exception: + pass + save_feeds(DEFAULT_FEEDS) + return [dict(f) for f in DEFAULT_FEEDS] + +def save_feeds(feeds): + _atomic_write_json(FEEDS_JSON, feeds) + +def add_feed(key, name, rss, n=2): + feeds = load_feeds() + feeds = [f for f in feeds if f["key"] != key] + feeds.append({"key": key, "name": name, "rss": rss, "n": int(n)}) + save_feeds(feeds) + return feeds + +def remove_feed(key): + save_feeds([f for f in load_feeds() if f["key"] != key]) + +# ---- RSS ---- +def cover_url(root): + el = root.find(".//channel/itunes:image", NS) + if el is not None and el.get("href"): + return el.get("href") + el = root.find(".//channel/image/url") + return el.text if el is not None else None + +def fetch_episodes(rss, limit=None): + """Return ([{url,title,duration,guid}], cover_href).""" + root = ET.fromstring(fetch(rss)) + items = root.findall(".//item") + if limit: + items = items[:limit] + out = [] + for it in items: + enc = it.find("enclosure") + if enc is None or not enc.get("url"): + continue + out.append({ + "url": enc.get("url"), + "title": clean(it.findtext("title") or "Episode"), + "duration": (it.findtext("itunes:duration", namespaces=NS) or "").strip(), + "guid": (it.findtext("guid") or enc.get("url")), + }) + return out, cover_url(root) + +# ---- transcode / cover ---- +def episode_fname(key, url): + return f"{key}_{hashlib.sha1(url.encode()).hexdigest()[:8]}_a48.mp3" + +def transcode(raw_path, out_path): + rc = subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", raw_path, + "-ac", "1", "-b:a", "48k", "-c:a", "libmp3lame", out_path], + capture_output=True).returncode + return rc == 0 and os.path.exists(out_path) + +def make_cover(img_bytes, key): + """160x160 RGB565 .bin into STAGING via LVGLImage. Returns '.bin' or ''.""" + try: + im = Image.open(io.BytesIO(img_bytes)).convert("RGB").resize((160, 160)) + os.makedirs(STAGING, exist_ok=True) + with tempfile.TemporaryDirectory() as td: + png = os.path.join(td, f"{key}.png"); im.save(png) + subprocess.run([sys.executable, LVGLIMG, "--ofmt", "BIN", "--cf", "RGB565", + "-o", td, png], check=True, capture_output=True) + binp = os.path.join(td, f"{key}.bin") + if os.path.exists(binp): + os.replace(binp, os.path.join(STAGING, f"{key}.bin")) + return f"{key}.bin" + except Exception as e: + print("cover fail", key, e, flush=True) + return "" + +# ---- staging / manifest ---- +def staging_files(): + if not os.path.isdir(STAGING): + return [] + return sorted(f for f in os.listdir(STAGING) if not f.startswith(".")) + +def manifest_load(): + p = os.path.join(STAGING, "manifest.json") + if os.path.exists(p): + try: + return json.load(open(p, encoding="utf-8")) + except Exception: + pass + return [] + +def manifest_save(m): + _atomic_write_json(os.path.join(STAGING, "manifest.json"), m) + +def _manifest_upsert(m, show, cover, file, title): + for s in m: + if s["show"] == show: + if cover and not s.get("cover"): + s["cover"] = cover + for e in s["episodes"]: + if e["file"] == file: + e["title"] = title + return + s["episodes"].append({"file": file, "title": title}) + return + m.append({"show": show, "cover": cover or "", "episodes": [{"file": file, "title": title}]}) + +def add_episode(key, show, cover_bytes, url, title): + """Download + transcode an episode into STAGING + upsert manifest. Idempotent by fname. + cover_bytes: raw image bytes for the show cover, or None.""" + os.makedirs(STAGING, exist_ok=True) + fname = episode_fname(key, url) + out = os.path.join(STAGING, fname) + if not os.path.exists(out): + with tempfile.TemporaryDirectory() as td: + raw = os.path.join(td, "raw"); open(raw, "wb").write(fetch(url)) + tmp_out = os.path.join(td, fname) + if not transcode(raw, tmp_out): + raise RuntimeError(f"transcode failed {fname}") + os.replace(tmp_out, out) + cover = make_cover(cover_bytes, key) if cover_bytes else "" + m = manifest_load() + _manifest_upsert(m, show, cover, fname, clean(title)) + manifest_save(m) + return fname + +def add_upload(show, src_bytes, title, key="perso"): + os.makedirs(STAGING, exist_ok=True) + h = hashlib.sha1(src_bytes[:65536] + str(len(src_bytes)).encode()).hexdigest()[:8] + fname = f"{key}_{h}_a48.mp3" + out = os.path.join(STAGING, fname) + with tempfile.TemporaryDirectory() as td: + raw = os.path.join(td, "raw"); open(raw, "wb").write(src_bytes) + tmp_out = os.path.join(td, fname) + if not transcode(raw, tmp_out): + raise RuntimeError("transcode failed") + os.replace(tmp_out, out) + m = manifest_load() + _manifest_upsert(m, show, "", fname, clean(title)) + manifest_save(m) + return fname + +def remove_file(name): + if "/" in name or ".." in name or name.startswith("."): + raise ValueError("bad name") + p = os.path.join(STAGING, name) + if os.path.exists(p): + os.remove(p) + m = manifest_load() + for s in m: + s["episodes"] = [e for e in s["episodes"] if e["file"] != name] + m = [s for s in m if s["episodes"]] + manifest_save(m) + +# ---- box state + push ---- +def box_state_load(): + if os.path.exists(BOX_JSON): + try: + return json.load(open(BOX_JSON, encoding="utf-8")) + except Exception: + pass + return {} + +def box_state_save(st): + _atomic_write_json(BOX_JSON, st) + +def compute_delta(staging, have): + haveset = set(have) + files = [f for f in staging if f != "manifest.json" and f not in haveset] + if files and "manifest.json" in staging: + files.append("manifest.json") + return files + +def push_file(ip, name): + data = open(os.path.join(STAGING, name), "rb").read() + url = f"http://{ip}:{BOX_PORT}/put?name=podcasts/{name}" + with urllib.request.urlopen(urllib.request.Request(url, data=data, method="POST"), + timeout=300) as r: + return r.status + +_busy = set() +_busy_lock = threading.Lock() + +def push_batch(ip, delta): + ok = 0 + for f in delta: + try: + push_file(ip, f); ok += 1 + except Exception as e: + print("push fail", f, e, flush=True) + print(f"push done {ip} {ok}/{len(delta)}", flush=True) + with _busy_lock: + _busy.discard(ip) + +def try_push(ip, delta): + """Start a background push unless one already runs for ip. Returns started?""" + if not delta: + return False + with _busy_lock: + if ip in _busy: + return False + _busy.add(ip) + threading.Thread(target=push_batch, args=(ip, delta), daemon=True).start() + return True diff --git a/tower/test_content.py b/tower/test_content.py new file mode 100644 index 0000000..a51d82b --- /dev/null +++ b/tower/test_content.py @@ -0,0 +1,48 @@ +import os, tempfile, json +os.environ["LISAEL_BASE"] = tempfile.mkdtemp(prefix="merlin_test_") +os.environ["LISAEL_STAGING"] = os.path.join(os.environ["LISAEL_BASE"], "podcasts") +import lisael_content as C + +def test_episode_fname_stable(): + a = C.episode_fname("oli", "http://x/ep1.m4a") + assert a == C.episode_fname("oli", "http://x/ep1.m4a") + assert a.endswith("_a48.mp3") and a.startswith("oli_") + assert a != C.episode_fname("oli", "http://x/ep2.m4a") + +def test_feeds_default_seed(): + feeds = C.load_feeds() + assert any(f["key"] == "oli" for f in feeds) + assert os.path.exists(C.FEEDS_JSON) + C.add_feed("zz", "Test", "http://r/ss", 1) + assert any(f["key"] == "zz" for f in C.load_feeds()) + C.remove_feed("zz") + assert not any(f["key"] == "zz" for f in C.load_feeds()) + +def test_manifest_upsert_remove(): + os.makedirs(C.STAGING, exist_ok=True) + m = [] + C._manifest_upsert(m, "Oli", "oli.bin", "oli_a_a48.mp3", "Ep A") + C._manifest_upsert(m, "Oli", "", "oli_b_a48.mp3", "Ep B") + assert len(m) == 1 and len(m[0]["episodes"]) == 2 and m[0]["cover"] == "oli.bin" + C.manifest_save(m) + open(os.path.join(C.STAGING, "oli_a_a48.mp3"), "wb").close() + C.remove_file("oli_a_a48.mp3") + m2 = C.manifest_load() + assert [e["file"] for e in m2[0]["episodes"]] == ["oli_b_a48.mp3"] + +def test_compute_delta(): + assert C.compute_delta([], []) == [] + assert set(C.compute_delta(["a.mp3", "manifest.json"], [])) == {"a.mp3", "manifest.json"} + assert C.compute_delta(["a.mp3", "manifest.json"], ["a.mp3"]) == [] + +def test_bad_name_rejected(): + try: + C.remove_file("../x"); assert False + except ValueError: + pass + +if __name__ == "__main__": + for name, fn in sorted(globals().items()): + if name.startswith("test_"): + fn(); print("ok", name) + print("ALL OK")