refactor(tower): additive refresh on shared module
This commit is contained in:
Clément Saillant
+23
-100
@@ -1,111 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Weekly refresh: latest Radio France youth episodes -> staging dir that
|
||||
lisael_server.py pushes to the box. Atomic swap so a failed run keeps the old set."""
|
||||
import hashlib, io, json, os, shutil, ssl, subprocess, sys, tempfile, urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from PIL import Image
|
||||
|
||||
STAGING = os.environ.get("LISAEL_STAGING", "/home/clems/lisael-content/podcasts")
|
||||
LVGLIMG = os.environ.get("LISAEL_LVGLIMG", "/home/clems/lisael-content/LVGLImage.py")
|
||||
UA = {"User-Agent": "Mozilla/5.0"}
|
||||
NS = {"itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd"}
|
||||
FEEDS = {
|
||||
"oli": ("Une histoire et Oli", "https://radiofrance-podcast.net/podcast09/rss_19721.xml", 3),
|
||||
"odyssees": ("Les Odyssees", "https://radiofrance-podcast.net/podcast09/rss_20108.xml", 3),
|
||||
"salutinfo": ("Salut l'info", "https://radiofrance-podcast.net/podcast09/rss_20689.xml", 2),
|
||||
"bestioles": ("Bestioles",
|
||||
"https://radiofrance-podcast.net/podcast09/35099478-7c72-4f9e-a6de-1b928400e9e5/"
|
||||
"podcast_a80ecbd5-df3d-4c9d-bee7-4e3d9efc1974.xml", 2),
|
||||
}
|
||||
SUBS = {"’": "'", "“": '"', "”": '"', "–": "-",
|
||||
"—": "-", "…": "...", "«": '"', "»": '"'}
|
||||
|
||||
def fetch(u, t=180):
|
||||
return urllib.request.urlopen(urllib.request.Request(u, headers=UA),
|
||||
timeout=t, context=ssl.create_default_context()).read()
|
||||
|
||||
def ext_for(url, mime):
|
||||
u = url.split("?")[0].lower()
|
||||
return "m4a" if (u.endswith(".m4a") or "mp4" in (mime or "") or "x-m4a" in (mime or "")) else "mp3"
|
||||
|
||||
def cover_url(root):
|
||||
el = root.find(".//channel/itunes:image", NS)
|
||||
if el is not None and el.get("href"):
|
||||
return el.get("href")
|
||||
el = root.find(".//channel/image/url")
|
||||
return el.text if el is not None else None
|
||||
|
||||
def clean(s):
|
||||
for a, b in SUBS.items():
|
||||
s = s.replace(a, b)
|
||||
return " ".join(s.split()).strip()
|
||||
"""Weekly refresh: for each configured feed, add the latest N missing episodes to
|
||||
the staging set. ADDITIVE — never wipes manual selections or uploads."""
|
||||
import lisael_content as C
|
||||
|
||||
def main():
|
||||
tmp = tempfile.mkdtemp(prefix="lisael_stg_")
|
||||
manifest = []
|
||||
for key, (show, rss, n) in FEEDS.items():
|
||||
added = 0
|
||||
for fd in C.load_feeds():
|
||||
key, show, rss, n = fd["key"], fd["name"], fd["rss"], int(fd.get("n", 2))
|
||||
try:
|
||||
root = ET.fromstring(fetch(rss))
|
||||
eps, cov = C.fetch_episodes(rss, limit=n)
|
||||
except Exception as e:
|
||||
print("RSS fail", key, e); continue
|
||||
eps = []
|
||||
for i, it in enumerate(root.findall(".//item")[:n], 1):
|
||||
enc = it.find("enclosure")
|
||||
if enc is None or not enc.get("url"):
|
||||
print("RSS fail", key, e, flush=True); continue
|
||||
cover_bytes = None
|
||||
if cov:
|
||||
try:
|
||||
cover_bytes = C.fetch(cov)
|
||||
except Exception:
|
||||
cover_bytes = None
|
||||
staged = set(C.staging_files())
|
||||
for ep in eps:
|
||||
if C.episode_fname(key, ep["url"]) in staged:
|
||||
continue
|
||||
url = enc.get("url")
|
||||
# Unique per episode (enclosure URL is stable+unique) so a new episode
|
||||
# = a new filename = the box detects it as missing. Positional names
|
||||
# (oli_01) would hide a new "latest" episode behind the same name.
|
||||
# The "_a48" tag marks the transcode profile: a 20+ MB original would
|
||||
# take minutes over the box's ~100 KB/s link, so we re-encode to mono
|
||||
# 48 kbps MP3 (plenty for spoken-word kids podcasts) — files drop to a
|
||||
# few MB and each push completes well under the server timeout. The tag
|
||||
# also gives the file a fresh name, so the box re-pulls the light
|
||||
# version and the old heavy original becomes an unreferenced orphan.
|
||||
h = hashlib.sha1(url.encode()).hexdigest()[:8]
|
||||
fname = f"{key}_{h}_a48.mp3"
|
||||
raw = os.path.join(tmp, f".raw_{key}_{h}")
|
||||
try:
|
||||
open(raw, "wb").write(fetch(url))
|
||||
C.add_episode(key, show, cover_bytes, ep["url"], ep["title"])
|
||||
cover_bytes = None # cover only needs writing once per show
|
||||
added += 1
|
||||
print("added", C.episode_fname(key, ep["url"]), flush=True)
|
||||
except Exception as e:
|
||||
print("dl fail", fname, e); continue
|
||||
out = os.path.join(tmp, fname)
|
||||
rc = subprocess.run(
|
||||
["ffmpeg", "-y", "-loglevel", "error", "-i", raw,
|
||||
"-ac", "1", "-b:a", "48k", "-c:a", "libmp3lame", out],
|
||||
capture_output=True).returncode
|
||||
try:
|
||||
os.remove(raw)
|
||||
except OSError:
|
||||
pass
|
||||
if rc != 0 or not os.path.exists(out):
|
||||
print("transcode fail", fname); continue
|
||||
eps.append({"file": fname, "title": clean(it.findtext("title") or f"{show} {i}")})
|
||||
if not eps:
|
||||
continue
|
||||
cover = ""
|
||||
try:
|
||||
cu = cover_url(root)
|
||||
if cu:
|
||||
im = Image.open(io.BytesIO(fetch(cu))).convert("RGB").resize((160, 160))
|
||||
png = os.path.join(tmp, f"{key}.png"); im.save(png)
|
||||
subprocess.run([sys.executable, LVGLIMG, "--ofmt", "BIN", "--cf", "RGB565",
|
||||
"-o", tmp, png], check=True)
|
||||
cover = f"{key}.bin"; os.remove(png)
|
||||
except Exception as e:
|
||||
print("cover fail", key, e)
|
||||
manifest.append({"show": show, "cover": cover, "episodes": eps})
|
||||
|
||||
json.dump(manifest, open(os.path.join(tmp, "manifest.json"), "w", encoding="utf-8"),
|
||||
ensure_ascii=False, indent=2)
|
||||
os.makedirs(os.path.dirname(STAGING), exist_ok=True)
|
||||
bak = STAGING + ".old"
|
||||
if os.path.exists(STAGING):
|
||||
shutil.rmtree(bak, ignore_errors=True); os.rename(STAGING, bak)
|
||||
os.rename(tmp, STAGING)
|
||||
shutil.rmtree(bak, ignore_errors=True)
|
||||
print(f"staging OK: {len(manifest)} shows, {len(os.listdir(STAGING))} files in {STAGING}")
|
||||
print("add fail", key, e, flush=True)
|
||||
print(f"refresh done: +{added} episodes, {len(C.staging_files())} files", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user