Clément Saillant
ec2dd3f24f
The additive refresh accumulated episodes without bound (Oli reached 13). Tag each manifest episode with a source (auto/pick/upload); the refresh now prunes "auto" episodes that aged out of the feed's latest N while keeping hand-picked and uploaded ones. Feeds default to N=5.
280 lines
10 KiB
Python
280 lines
10 KiB
Python
#!/usr/bin/env python3
|
||
"""Shared content logic for the Lisael Box on Tower: feeds config, RSS, transcode,
|
||
covers, staging/manifest ops, box state, push. Imported by lisael_refresh.py (cron)
|
||
and lisael_server.py (web)."""
|
||
import hashlib, io, json, os, ssl, subprocess, sys, tempfile, threading, urllib.request
|
||
import xml.etree.ElementTree as ET
|
||
from PIL import Image
|
||
|
||
BASE = os.environ.get("LISAEL_BASE", "/home/clems/lisael-content")
|
||
STAGING = os.environ.get("LISAEL_STAGING", os.path.join(BASE, "podcasts"))
|
||
LVGLIMG = os.environ.get("LISAEL_LVGLIMG", os.path.join(BASE, "LVGLImage.py"))
|
||
FEEDS_JSON = os.path.join(BASE, "feeds.json")
|
||
BOX_JSON = os.path.join(BASE, "box-state.json")
|
||
BOX_PORT = int(os.environ.get("LISAEL_BOX_PORT", "8080"))
|
||
UA = {"User-Agent": "Mozilla/5.0"}
|
||
NS = {"itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd"}
|
||
SUBS = {"’": "'", "“": '"', "”": '"', "–": "-",
|
||
"—": "-", "…": "...", "«": '"', "»": '"'}
|
||
|
||
DEFAULT_FEEDS = [
|
||
{"key": "oli", "name": "Une histoire et Oli", "rss": "https://radiofrance-podcast.net/podcast09/rss_19721.xml", "n": 3},
|
||
{"key": "odyssees", "name": "Les Odyssees", "rss": "https://radiofrance-podcast.net/podcast09/rss_20108.xml", "n": 3},
|
||
{"key": "salutinfo", "name": "Salut l'info", "rss": "https://radiofrance-podcast.net/podcast09/rss_20689.xml", "n": 2},
|
||
{"key": "bestioles", "name": "Bestioles", "rss": "https://radiofrance-podcast.net/podcast09/35099478-7c72-4f9e-a6de-1b928400e9e5/podcast_a80ecbd5-df3d-4c9d-bee7-4e3d9efc1974.xml", "n": 2},
|
||
]
|
||
|
||
def fetch(u, t=180):
|
||
return urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=t,
|
||
context=ssl.create_default_context()).read()
|
||
|
||
def clean(s):
|
||
for a, b in SUBS.items():
|
||
s = s.replace(a, b)
|
||
return " ".join(s.split()).strip()
|
||
|
||
def _atomic_write_bytes(path, data):
|
||
d = os.path.dirname(path) or "."
|
||
os.makedirs(d, exist_ok=True)
|
||
fd, tmp = tempfile.mkstemp(dir=d)
|
||
with os.fdopen(fd, "wb") as f:
|
||
f.write(data)
|
||
os.replace(tmp, path)
|
||
|
||
def _atomic_write_json(path, obj):
|
||
_atomic_write_bytes(path, json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8"))
|
||
|
||
# ---- feeds config ----
|
||
def load_feeds():
|
||
if os.path.exists(FEEDS_JSON):
|
||
try:
|
||
return json.load(open(FEEDS_JSON, encoding="utf-8"))
|
||
except Exception:
|
||
pass
|
||
save_feeds(DEFAULT_FEEDS)
|
||
return [dict(f) for f in DEFAULT_FEEDS]
|
||
|
||
def save_feeds(feeds):
|
||
_atomic_write_json(FEEDS_JSON, feeds)
|
||
|
||
def add_feed(key, name, rss, n=2):
|
||
feeds = load_feeds()
|
||
feeds = [f for f in feeds if f["key"] != key]
|
||
feeds.append({"key": key, "name": name, "rss": rss, "n": int(n)})
|
||
save_feeds(feeds)
|
||
return feeds
|
||
|
||
def remove_feed(key):
|
||
save_feeds([f for f in load_feeds() if f["key"] != key])
|
||
|
||
# ---- RSS ----
|
||
def cover_url(root):
|
||
el = root.find(".//channel/itunes:image", NS)
|
||
if el is not None and el.get("href"):
|
||
return el.get("href")
|
||
el = root.find(".//channel/image/url")
|
||
return el.text if el is not None else None
|
||
|
||
def fetch_episodes(rss, limit=None):
|
||
"""Return ([{url,title,duration,guid}], cover_href)."""
|
||
root = ET.fromstring(fetch(rss))
|
||
items = root.findall(".//item")
|
||
if limit:
|
||
items = items[:limit]
|
||
out = []
|
||
for it in items:
|
||
enc = it.find("enclosure")
|
||
if enc is None or not enc.get("url"):
|
||
continue
|
||
out.append({
|
||
"url": enc.get("url"),
|
||
"title": clean(it.findtext("title") or "Episode"),
|
||
"duration": (it.findtext("itunes:duration", namespaces=NS) or "").strip(),
|
||
"guid": (it.findtext("guid") or enc.get("url")),
|
||
})
|
||
return out, cover_url(root)
|
||
|
||
# ---- transcode / cover ----
|
||
def episode_fname(key, url):
|
||
return f"{key}_{hashlib.sha1(url.encode()).hexdigest()[:8]}_a48.mp3"
|
||
|
||
def transcode(raw_path, out_path):
|
||
rc = subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", raw_path,
|
||
"-ac", "1", "-b:a", "48k", "-c:a", "libmp3lame", out_path],
|
||
capture_output=True).returncode
|
||
return rc == 0 and os.path.exists(out_path)
|
||
|
||
def make_cover(img_bytes, key):
|
||
"""160x160 RGB565 .bin into STAGING via LVGLImage. Returns '<key>.bin' or ''."""
|
||
try:
|
||
im = Image.open(io.BytesIO(img_bytes)).convert("RGB").resize((160, 160))
|
||
os.makedirs(STAGING, exist_ok=True)
|
||
with tempfile.TemporaryDirectory() as td:
|
||
png = os.path.join(td, f"{key}.png"); im.save(png)
|
||
subprocess.run([sys.executable, LVGLIMG, "--ofmt", "BIN", "--cf", "RGB565",
|
||
"-o", td, png], check=True, capture_output=True)
|
||
binp = os.path.join(td, f"{key}.bin")
|
||
if os.path.exists(binp):
|
||
os.replace(binp, os.path.join(STAGING, f"{key}.bin"))
|
||
return f"{key}.bin"
|
||
except Exception as e:
|
||
print("cover fail", key, e, flush=True)
|
||
return ""
|
||
|
||
# ---- staging / manifest ----
|
||
def staging_files():
|
||
if not os.path.isdir(STAGING):
|
||
return []
|
||
return sorted(f for f in os.listdir(STAGING) if not f.startswith("."))
|
||
|
||
def manifest_load():
|
||
p = os.path.join(STAGING, "manifest.json")
|
||
if os.path.exists(p):
|
||
try:
|
||
return json.load(open(p, encoding="utf-8"))
|
||
except Exception:
|
||
pass
|
||
return []
|
||
|
||
def manifest_save(m):
|
||
_atomic_write_json(os.path.join(STAGING, "manifest.json"), m)
|
||
|
||
def _manifest_upsert(m, show, cover, file, title, src="auto"):
|
||
# src: "auto" (weekly refresh), "pick" (hand-selected), "upload" (custom file).
|
||
# Only "auto" episodes are eligible for pruning by the cap.
|
||
for s in m:
|
||
if s["show"] == show:
|
||
if cover and not s.get("cover"):
|
||
s["cover"] = cover
|
||
for e in s["episodes"]:
|
||
if e["file"] == file:
|
||
e["title"] = title
|
||
if src != "auto": # a manual pick/upload wins over auto
|
||
e["src"] = src
|
||
return
|
||
s["episodes"].append({"file": file, "title": title, "src": src})
|
||
return
|
||
m.append({"show": show, "cover": cover or "",
|
||
"episodes": [{"file": file, "title": title, "src": src}]})
|
||
|
||
def add_episode(key, show, cover_bytes, url, title, src="auto"):
|
||
"""Download + transcode an episode into STAGING + upsert manifest. Idempotent by fname.
|
||
cover_bytes: raw image bytes for the show cover, or None. src: auto|pick."""
|
||
os.makedirs(STAGING, exist_ok=True)
|
||
fname = episode_fname(key, url)
|
||
out = os.path.join(STAGING, fname)
|
||
if not os.path.exists(out):
|
||
with tempfile.TemporaryDirectory() as td:
|
||
raw = os.path.join(td, "raw"); open(raw, "wb").write(fetch(url))
|
||
tmp_out = os.path.join(td, fname)
|
||
if not transcode(raw, tmp_out):
|
||
raise RuntimeError(f"transcode failed {fname}")
|
||
os.replace(tmp_out, out)
|
||
cover = make_cover(cover_bytes, key) if cover_bytes else ""
|
||
m = manifest_load()
|
||
_manifest_upsert(m, show, cover, fname, clean(title), src)
|
||
manifest_save(m)
|
||
return fname
|
||
|
||
def add_upload(show, src_bytes, title, key="perso"):
|
||
os.makedirs(STAGING, exist_ok=True)
|
||
h = hashlib.sha1(src_bytes[:65536] + str(len(src_bytes)).encode()).hexdigest()[:8]
|
||
fname = f"{key}_{h}_a48.mp3"
|
||
out = os.path.join(STAGING, fname)
|
||
with tempfile.TemporaryDirectory() as td:
|
||
raw = os.path.join(td, "raw"); open(raw, "wb").write(src_bytes)
|
||
tmp_out = os.path.join(td, fname)
|
||
if not transcode(raw, tmp_out):
|
||
raise RuntimeError("transcode failed")
|
||
os.replace(tmp_out, out)
|
||
m = manifest_load()
|
||
_manifest_upsert(m, show, "", fname, clean(title), "upload")
|
||
manifest_save(m)
|
||
return fname
|
||
|
||
def prune_show(show, keep_files):
|
||
"""Cap a feed show: drop its 'auto' episodes whose file is not in keep_files
|
||
(the current latest-N from the feed). 'pick'/'upload' episodes are always kept.
|
||
Deletes the staged file too. Returns the number removed."""
|
||
m = manifest_load()
|
||
removed = 0
|
||
for s in m:
|
||
if s["show"] != show:
|
||
continue
|
||
kept = []
|
||
for e in s["episodes"]:
|
||
if e.get("src", "auto") != "auto" or e["file"] in keep_files:
|
||
kept.append(e)
|
||
else:
|
||
p = os.path.join(STAGING, e["file"])
|
||
if os.path.exists(p):
|
||
os.remove(p)
|
||
removed += 1
|
||
s["episodes"] = kept
|
||
m = [s for s in m if s["episodes"]]
|
||
manifest_save(m)
|
||
return removed
|
||
|
||
def remove_file(name):
|
||
if "/" in name or ".." in name or name.startswith("."):
|
||
raise ValueError("bad name")
|
||
p = os.path.join(STAGING, name)
|
||
if os.path.exists(p):
|
||
os.remove(p)
|
||
m = manifest_load()
|
||
for s in m:
|
||
s["episodes"] = [e for e in s["episodes"] if e["file"] != name]
|
||
m = [s for s in m if s["episodes"]]
|
||
manifest_save(m)
|
||
|
||
# ---- box state + push ----
|
||
def box_state_load():
|
||
if os.path.exists(BOX_JSON):
|
||
try:
|
||
return json.load(open(BOX_JSON, encoding="utf-8"))
|
||
except Exception:
|
||
pass
|
||
return {}
|
||
|
||
def box_state_save(st):
|
||
_atomic_write_json(BOX_JSON, st)
|
||
|
||
def compute_delta(staging, have):
|
||
haveset = set(have)
|
||
files = [f for f in staging if f != "manifest.json" and f not in haveset]
|
||
if files and "manifest.json" in staging:
|
||
files.append("manifest.json")
|
||
return files
|
||
|
||
def push_file(ip, name):
|
||
data = open(os.path.join(STAGING, name), "rb").read()
|
||
url = f"http://{ip}:{BOX_PORT}/put?name=podcasts/{name}"
|
||
with urllib.request.urlopen(urllib.request.Request(url, data=data, method="POST"),
|
||
timeout=300) as r:
|
||
return r.status
|
||
|
||
_busy = set()
|
||
_busy_lock = threading.Lock()
|
||
|
||
def push_batch(ip, delta):
|
||
ok = 0
|
||
for f in delta:
|
||
try:
|
||
push_file(ip, f); ok += 1
|
||
except Exception as e:
|
||
print("push fail", f, e, flush=True)
|
||
print(f"push done {ip} {ok}/{len(delta)}", flush=True)
|
||
with _busy_lock:
|
||
_busy.discard(ip)
|
||
|
||
def try_push(ip, delta):
|
||
"""Start a background push unless one already runs for ip. Returns started?"""
|
||
if not delta:
|
||
return False
|
||
with _busy_lock:
|
||
if ip in _busy:
|
||
return False
|
||
_busy.add(ip)
|
||
threading.Thread(target=push_batch, args=(ip, delta), daemon=True).start()
|
||
return True
|