docs: Merlin Studio design + plan
This commit is contained in:
@@ -0,0 +1,663 @@
|
||||
# Merlin Studio Implementation Plan
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Page web `merlin.saillant.cc` (sur Tower) pour curer les podcasts de la Lisael Box, gérer les fichiers du staging, et piloter la box.
|
||||
|
||||
**Architecture:** Un module partagé `lisael_content.py` (RSS, transcode, manifeste, push, config, box-state) utilisé par le cron `lisael_refresh.py` et le serveur web `lisael_server.py` (qui sert aussi l'UI `merlin_ui.html` + `/api/*`, derrière Basic Auth). Tout vit dans `/home/clems/lisael-content/` sur Tower.
|
||||
|
||||
**Tech Stack:** Python 3 stdlib (http.server), Pillow + LVGLImage + ffmpeg, HTML/CSS/JS vanilla. Fichiers dans `tower/` du repo `lisael-box`.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: Module partagé `lisael_content.py`
|
||||
|
||||
**Files:**
|
||||
- Create: `tower/lisael_content.py`
|
||||
- Test: `tower/test_content.py`
|
||||
|
||||
- [ ] **Step 1: Écrire `tower/lisael_content.py`** (contenu complet ci-dessous)
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
"""Shared content logic for the Lisael Box on Tower: feeds config, RSS, transcode,
|
||||
covers, staging/manifest ops, box state, push. Imported by lisael_refresh.py (cron)
|
||||
and lisael_server.py (web)."""
|
||||
import hashlib, io, json, os, ssl, subprocess, sys, tempfile, threading, urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from PIL import Image
|
||||
|
||||
BASE = os.environ.get("LISAEL_BASE", "/home/clems/lisael-content")
|
||||
STAGING = os.environ.get("LISAEL_STAGING", os.path.join(BASE, "podcasts"))
|
||||
LVGLIMG = os.environ.get("LISAEL_LVGLIMG", os.path.join(BASE, "LVGLImage.py"))
|
||||
FEEDS_JSON = os.path.join(BASE, "feeds.json")
|
||||
BOX_JSON = os.path.join(BASE, "box-state.json")
|
||||
BOX_PORT = int(os.environ.get("LISAEL_BOX_PORT", "8080"))
|
||||
UA = {"User-Agent": "Mozilla/5.0"}
|
||||
NS = {"itunes": "http://www.itunes.com/dtds/podcast-1.0.dtd"}
|
||||
SUBS = {"’": "'", "“": '"', "”": '"', "–": "-",
|
||||
"—": "-", "…": "...", "«": '"', "»": '"'}
|
||||
|
||||
DEFAULT_FEEDS = [
|
||||
{"key": "oli", "name": "Une histoire et Oli", "rss": "https://radiofrance-podcast.net/podcast09/rss_19721.xml", "n": 3},
|
||||
{"key": "odyssees", "name": "Les Odyssees", "rss": "https://radiofrance-podcast.net/podcast09/rss_20108.xml", "n": 3},
|
||||
{"key": "salutinfo", "name": "Salut l'info", "rss": "https://radiofrance-podcast.net/podcast09/rss_20689.xml", "n": 2},
|
||||
{"key": "bestioles", "name": "Bestioles", "rss": "https://radiofrance-podcast.net/podcast09/35099478-7c72-4f9e-a6de-1b928400e9e5/podcast_a80ecbd5-df3d-4c9d-bee7-4e3d9efc1974.xml", "n": 2},
|
||||
]
|
||||
|
||||
def fetch(u, t=180):
|
||||
return urllib.request.urlopen(urllib.request.Request(u, headers=UA), timeout=t,
|
||||
context=ssl.create_default_context()).read()
|
||||
|
||||
def clean(s):
|
||||
for a, b in SUBS.items():
|
||||
s = s.replace(a, b)
|
||||
return " ".join(s.split()).strip()
|
||||
|
||||
def _atomic_write_bytes(path, data):
|
||||
d = os.path.dirname(path) or "."
|
||||
os.makedirs(d, exist_ok=True)
|
||||
fd, tmp = tempfile.mkstemp(dir=d)
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(data)
|
||||
os.replace(tmp, path)
|
||||
|
||||
def _atomic_write_json(path, obj):
|
||||
_atomic_write_bytes(path, json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8"))
|
||||
|
||||
# ---- feeds config ----
|
||||
def load_feeds():
|
||||
if os.path.exists(FEEDS_JSON):
|
||||
try:
|
||||
return json.load(open(FEEDS_JSON, encoding="utf-8"))
|
||||
except Exception:
|
||||
pass
|
||||
save_feeds(DEFAULT_FEEDS)
|
||||
return [dict(f) for f in DEFAULT_FEEDS]
|
||||
|
||||
def save_feeds(feeds):
|
||||
_atomic_write_json(FEEDS_JSON, feeds)
|
||||
|
||||
def add_feed(key, name, rss, n=2):
|
||||
feeds = load_feeds()
|
||||
feeds = [f for f in feeds if f["key"] != key]
|
||||
feeds.append({"key": key, "name": name, "rss": rss, "n": int(n)})
|
||||
save_feeds(feeds)
|
||||
return feeds
|
||||
|
||||
def remove_feed(key):
|
||||
save_feeds([f for f in load_feeds() if f["key"] != key])
|
||||
|
||||
# ---- RSS ----
|
||||
def cover_url(root):
|
||||
el = root.find(".//channel/itunes:image", NS)
|
||||
if el is not None and el.get("href"):
|
||||
return el.get("href")
|
||||
el = root.find(".//channel/image/url")
|
||||
return el.text if el is not None else None
|
||||
|
||||
def fetch_episodes(rss, limit=None):
|
||||
"""Return ([{url,title,duration,guid}], cover_href)."""
|
||||
root = ET.fromstring(fetch(rss))
|
||||
items = root.findall(".//item")
|
||||
if limit:
|
||||
items = items[:limit]
|
||||
out = []
|
||||
for it in items:
|
||||
enc = it.find("enclosure")
|
||||
if enc is None or not enc.get("url"):
|
||||
continue
|
||||
out.append({
|
||||
"url": enc.get("url"),
|
||||
"title": clean(it.findtext("title") or "Episode"),
|
||||
"duration": (it.findtext("itunes:duration", namespaces=NS) or "").strip(),
|
||||
"guid": (it.findtext("guid") or enc.get("url")),
|
||||
})
|
||||
return out, cover_url(root)
|
||||
|
||||
# ---- transcode / cover ----
|
||||
def episode_fname(key, url):
|
||||
return f"{key}_{hashlib.sha1(url.encode()).hexdigest()[:8]}_a48.mp3"
|
||||
|
||||
def transcode(raw_path, out_path):
|
||||
rc = subprocess.run(["ffmpeg", "-y", "-loglevel", "error", "-i", raw_path,
|
||||
"-ac", "1", "-b:a", "48k", "-c:a", "libmp3lame", out_path],
|
||||
capture_output=True).returncode
|
||||
return rc == 0 and os.path.exists(out_path)
|
||||
|
||||
def make_cover(img_bytes, key):
|
||||
"""160x160 RGB565 .bin into STAGING via LVGLImage. Returns '<key>.bin' or ''."""
|
||||
try:
|
||||
im = Image.open(io.BytesIO(img_bytes)).convert("RGB").resize((160, 160))
|
||||
os.makedirs(STAGING, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
png = os.path.join(td, f"{key}.png"); im.save(png)
|
||||
subprocess.run([sys.executable, LVGLIMG, "--ofmt", "BIN", "--cf", "RGB565",
|
||||
"-o", td, png], check=True, capture_output=True)
|
||||
binp = os.path.join(td, f"{key}.bin")
|
||||
if os.path.exists(binp):
|
||||
os.replace(binp, os.path.join(STAGING, f"{key}.bin"))
|
||||
return f"{key}.bin"
|
||||
except Exception as e:
|
||||
print("cover fail", key, e, flush=True)
|
||||
return ""
|
||||
|
||||
# ---- staging / manifest ----
|
||||
def staging_files():
|
||||
if not os.path.isdir(STAGING):
|
||||
return []
|
||||
return sorted(f for f in os.listdir(STAGING) if not f.startswith("."))
|
||||
|
||||
def manifest_load():
|
||||
p = os.path.join(STAGING, "manifest.json")
|
||||
if os.path.exists(p):
|
||||
try:
|
||||
return json.load(open(p, encoding="utf-8"))
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
def manifest_save(m):
|
||||
_atomic_write_json(os.path.join(STAGING, "manifest.json"), m)
|
||||
|
||||
def _manifest_upsert(m, show, cover, file, title):
|
||||
for s in m:
|
||||
if s["show"] == show:
|
||||
if cover and not s.get("cover"):
|
||||
s["cover"] = cover
|
||||
for e in s["episodes"]:
|
||||
if e["file"] == file:
|
||||
e["title"] = title
|
||||
return
|
||||
s["episodes"].append({"file": file, "title": title})
|
||||
return
|
||||
m.append({"show": show, "cover": cover or "", "episodes": [{"file": file, "title": title}]})
|
||||
|
||||
def add_episode(key, show, cover_bytes, url, title):
|
||||
"""Download + transcode an episode into STAGING + upsert manifest. Idempotent by fname.
|
||||
cover_bytes: raw image bytes for the show cover, or None."""
|
||||
os.makedirs(STAGING, exist_ok=True)
|
||||
fname = episode_fname(key, url)
|
||||
out = os.path.join(STAGING, fname)
|
||||
if not os.path.exists(out):
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
raw = os.path.join(td, "raw"); open(raw, "wb").write(fetch(url))
|
||||
tmp_out = os.path.join(td, fname)
|
||||
if not transcode(raw, tmp_out):
|
||||
raise RuntimeError(f"transcode failed {fname}")
|
||||
os.replace(tmp_out, out)
|
||||
cover = make_cover(cover_bytes, key) if cover_bytes else ""
|
||||
m = manifest_load()
|
||||
_manifest_upsert(m, show, cover, fname, clean(title))
|
||||
manifest_save(m)
|
||||
return fname
|
||||
|
||||
def add_upload(show, src_bytes, title, key="perso"):
|
||||
os.makedirs(STAGING, exist_ok=True)
|
||||
h = hashlib.sha1(src_bytes[:65536] + str(len(src_bytes)).encode()).hexdigest()[:8]
|
||||
fname = f"{key}_{h}_a48.mp3"
|
||||
out = os.path.join(STAGING, fname)
|
||||
with tempfile.TemporaryDirectory() as td:
|
||||
raw = os.path.join(td, "raw"); open(raw, "wb").write(src_bytes)
|
||||
tmp_out = os.path.join(td, fname)
|
||||
if not transcode(raw, tmp_out):
|
||||
raise RuntimeError("transcode failed")
|
||||
os.replace(tmp_out, out)
|
||||
m = manifest_load()
|
||||
_manifest_upsert(m, show, "", fname, clean(title))
|
||||
manifest_save(m)
|
||||
return fname
|
||||
|
||||
def remove_file(name):
|
||||
if "/" in name or ".." in name or name.startswith("."):
|
||||
raise ValueError("bad name")
|
||||
p = os.path.join(STAGING, name)
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
m = manifest_load()
|
||||
for s in m:
|
||||
s["episodes"] = [e for e in s["episodes"] if e["file"] != name]
|
||||
m = [s for s in m if s["episodes"]]
|
||||
manifest_save(m)
|
||||
|
||||
# ---- box state + push ----
|
||||
def box_state_load():
|
||||
if os.path.exists(BOX_JSON):
|
||||
try:
|
||||
return json.load(open(BOX_JSON, encoding="utf-8"))
|
||||
except Exception:
|
||||
pass
|
||||
return {}
|
||||
|
||||
def box_state_save(st):
|
||||
_atomic_write_json(BOX_JSON, st)
|
||||
|
||||
def compute_delta(staging, have):
|
||||
haveset = set(have)
|
||||
files = [f for f in staging if f != "manifest.json" and f not in haveset]
|
||||
if files and "manifest.json" in staging:
|
||||
files.append("manifest.json")
|
||||
return files
|
||||
|
||||
def push_file(ip, name):
|
||||
data = open(os.path.join(STAGING, name), "rb").read()
|
||||
url = f"http://{ip}:{BOX_PORT}/put?name=podcasts/{name}"
|
||||
with urllib.request.urlopen(urllib.request.Request(url, data=data, method="POST"),
|
||||
timeout=300) as r:
|
||||
return r.status
|
||||
|
||||
_busy = set()
|
||||
_busy_lock = threading.Lock()
|
||||
|
||||
def push_batch(ip, delta):
|
||||
ok = 0
|
||||
for f in delta:
|
||||
try:
|
||||
push_file(ip, f); ok += 1
|
||||
except Exception as e:
|
||||
print("push fail", f, e, flush=True)
|
||||
print(f"push done {ip} {ok}/{len(delta)}", flush=True)
|
||||
with _busy_lock:
|
||||
_busy.discard(ip)
|
||||
|
||||
def try_push(ip, delta):
|
||||
"""Start a background push unless one already runs for ip. Returns started?"""
|
||||
if not delta:
|
||||
return False
|
||||
with _busy_lock:
|
||||
if ip in _busy:
|
||||
return False
|
||||
_busy.add(ip)
|
||||
threading.Thread(target=push_batch, args=(ip, delta), daemon=True).start()
|
||||
return True
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Écrire `tower/test_content.py`** (tests hôte, sans réseau)
|
||||
|
||||
```python
|
||||
import os, tempfile, json
|
||||
os.environ["LISAEL_BASE"] = tempfile.mkdtemp(prefix="merlin_test_")
|
||||
os.environ["LISAEL_STAGING"] = os.path.join(os.environ["LISAEL_BASE"], "podcasts")
|
||||
import lisael_content as C
|
||||
|
||||
def test_episode_fname_stable():
|
||||
a = C.episode_fname("oli", "http://x/ep1.m4a")
|
||||
assert a == C.episode_fname("oli", "http://x/ep1.m4a")
|
||||
assert a.endswith("_a48.mp3") and a.startswith("oli_")
|
||||
assert a != C.episode_fname("oli", "http://x/ep2.m4a")
|
||||
|
||||
def test_feeds_default_seed():
|
||||
feeds = C.load_feeds()
|
||||
assert any(f["key"] == "oli" for f in feeds)
|
||||
assert os.path.exists(C.FEEDS_JSON)
|
||||
C.add_feed("zz", "Test", "http://r/ss", 1)
|
||||
assert any(f["key"] == "zz" for f in C.load_feeds())
|
||||
C.remove_feed("zz")
|
||||
assert not any(f["key"] == "zz" for f in C.load_feeds())
|
||||
|
||||
def test_manifest_upsert_remove():
|
||||
os.makedirs(C.STAGING, exist_ok=True)
|
||||
m = []
|
||||
C._manifest_upsert(m, "Oli", "oli.bin", "oli_a_a48.mp3", "Ep A")
|
||||
C._manifest_upsert(m, "Oli", "", "oli_b_a48.mp3", "Ep B")
|
||||
assert len(m) == 1 and len(m[0]["episodes"]) == 2 and m[0]["cover"] == "oli.bin"
|
||||
C.manifest_save(m)
|
||||
open(os.path.join(C.STAGING, "oli_a_a48.mp3"), "wb").close()
|
||||
C.remove_file("oli_a_a48.mp3")
|
||||
m2 = C.manifest_load()
|
||||
assert [e["file"] for e in m2[0]["episodes"]] == ["oli_b_a48.mp3"]
|
||||
|
||||
def test_compute_delta():
|
||||
assert C.compute_delta([], []) == []
|
||||
assert set(C.compute_delta(["a.mp3", "manifest.json"], [])) == {"a.mp3", "manifest.json"}
|
||||
assert C.compute_delta(["a.mp3", "manifest.json"], ["a.mp3"]) == []
|
||||
|
||||
def test_bad_name_rejected():
|
||||
try:
|
||||
C.remove_file("../x"); assert False
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
for name, fn in sorted(globals().items()):
|
||||
if name.startswith("test_"):
|
||||
fn(); print("ok", name)
|
||||
print("ALL OK")
|
||||
```
|
||||
|
||||
- [ ] **Step 3: Lancer les tests** — `cd tower && python3 test_content.py` → attendu `ALL OK`.
|
||||
- [ ] **Step 4: Commit** — `git add tower/lisael_content.py tower/test_content.py && git commit -m "feat(tower): shared content module"`
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Refactor `lisael_refresh.py` (merge additif)
|
||||
|
||||
**Files:** Modify: `tower/lisael_refresh.py` (réécriture complète)
|
||||
|
||||
- [ ] **Step 1: Remplacer tout `tower/lisael_refresh.py` par :**
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
"""Weekly refresh: for each configured feed, add the latest N missing episodes to
|
||||
the staging set. ADDITIVE — never wipes manual selections or uploads."""
|
||||
import lisael_content as C
|
||||
|
||||
def main():
|
||||
added = 0
|
||||
for fd in C.load_feeds():
|
||||
key, show, rss, n = fd["key"], fd["name"], fd["rss"], int(fd.get("n", 2))
|
||||
try:
|
||||
eps, cov = C.fetch_episodes(rss, limit=n)
|
||||
except Exception as e:
|
||||
print("RSS fail", key, e, flush=True); continue
|
||||
cover_bytes = None
|
||||
if cov:
|
||||
try:
|
||||
cover_bytes = C.fetch(cov)
|
||||
except Exception:
|
||||
cover_bytes = None
|
||||
staged = set(C.staging_files())
|
||||
for ep in eps:
|
||||
if C.episode_fname(key, ep["url"]) in staged:
|
||||
continue
|
||||
try:
|
||||
C.add_episode(key, show, cover_bytes, ep["url"], ep["title"])
|
||||
cover_bytes = None # cover only needs writing once per show
|
||||
added += 1
|
||||
print("added", C.episode_fname(key, ep["url"]), flush=True)
|
||||
except Exception as e:
|
||||
print("add fail", key, e, flush=True)
|
||||
print(f"refresh done: +{added} episodes, {len(C.staging_files())} files", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Sanity** — `cd tower && python3 -c "import ast; ast.parse(open('lisael_refresh.py').read()); print('ok')"`.
|
||||
- [ ] **Step 3: Commit** — `git commit -am "refactor(tower): additive refresh on shared module"`
|
||||
|
||||
---
|
||||
|
||||
### Task 3: APIs admin dans `lisael_server.py`
|
||||
|
||||
**Files:** Modify: `tower/lisael_server.py` (réécriture complète)
|
||||
|
||||
- [ ] **Step 1: Remplacer tout `tower/lisael_server.py` par :**
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
"""Always-on Lisael Box content service on Tower:
|
||||
- POST /register {id,have} (machine: box announces, we push the delta; LAN/tailnet only)
|
||||
- Merlin Studio web UI (GET /) + /api/* (human, behind Basic Auth)
|
||||
"""
|
||||
import base64, ipaddress, json, os, threading, urllib.parse
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import lisael_content as C
|
||||
|
||||
PORT = int(os.environ.get("LISAEL_PORT", "9555"))
|
||||
UI_HTML = os.path.join(os.path.dirname(os.path.abspath(__file__)), "merlin_ui.html")
|
||||
AUTH = os.environ.get("MERLIN_AUTH", "") # "user:pass"; empty disables auth
|
||||
|
||||
def _is_private(ip):
|
||||
try:
|
||||
a = ipaddress.ip_address(ip)
|
||||
return a.is_private or a.is_loopback or ip.startswith("100.") # 100.x = tailnet
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
class H(BaseHTTPRequestHandler):
|
||||
# --- helpers ---
|
||||
def _json(self, obj, code=200):
|
||||
b = json.dumps(obj, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(code); self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(b))); self.end_headers()
|
||||
self.wfile.write(b)
|
||||
|
||||
def _auth_ok(self):
|
||||
if not AUTH:
|
||||
return True
|
||||
h = self.headers.get("Authorization", "")
|
||||
if h.startswith("Basic "):
|
||||
try:
|
||||
if base64.b64decode(h[6:]).decode() == AUTH:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
self.send_response(401)
|
||||
self.send_header("WWW-Authenticate", 'Basic realm="Merlin Studio"')
|
||||
self.end_headers()
|
||||
return False
|
||||
|
||||
def _body(self):
|
||||
n = int(self.headers.get("Content-Length", 0) or 0)
|
||||
return self.rfile.read(n) if n else b""
|
||||
|
||||
def log_message(self, *a):
|
||||
pass
|
||||
|
||||
# --- GET ---
|
||||
def do_GET(self):
|
||||
path = self.path.split("?")[0]
|
||||
if path == "/register":
|
||||
self.send_error(405); return
|
||||
if not self._auth_ok():
|
||||
return
|
||||
if path == "/" or path == "/index.html":
|
||||
try:
|
||||
data = open(UI_HTML, "rb").read()
|
||||
except OSError:
|
||||
self.send_error(500, "UI missing"); return
|
||||
self.send_response(200); self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(data))); self.end_headers()
|
||||
self.wfile.write(data); return
|
||||
if path == "/api/feeds":
|
||||
self._json(C.load_feeds()); return
|
||||
if path == "/api/episodes":
|
||||
q = urllib.parse.parse_qs(self.path.split("?", 1)[1] if "?" in self.path else "")
|
||||
key = (q.get("feed") or [""])[0]
|
||||
feed = next((f for f in C.load_feeds() if f["key"] == key), None)
|
||||
if not feed:
|
||||
self._json({"error": "unknown feed"}, 404); return
|
||||
try:
|
||||
eps, cov = C.fetch_episodes(feed["rss"])
|
||||
except Exception as e:
|
||||
self._json({"error": str(e)}, 502); return
|
||||
staged = set(C.staging_files())
|
||||
for e in eps:
|
||||
e["staged"] = C.episode_fname(key, e["url"]) in staged
|
||||
self._json({"feed": feed, "cover": cov, "episodes": eps}); return
|
||||
if path == "/api/files":
|
||||
out = []
|
||||
for f in C.staging_files():
|
||||
p = os.path.join(C.STAGING, f)
|
||||
out.append({"name": f, "size": os.path.getsize(p),
|
||||
"type": f.rsplit(".", 1)[-1] if "." in f else ""})
|
||||
self._json(out); return
|
||||
if path == "/api/box":
|
||||
st = C.box_state_load()
|
||||
delta = C.compute_delta(C.staging_files(), st.get("have", []))
|
||||
self._json({"box": st, "delta": delta, "staging_count": len(C.staging_files())}); return
|
||||
self.send_error(404)
|
||||
|
||||
# --- POST ---
|
||||
def do_POST(self):
|
||||
path = self.path.split("?")[0]
|
||||
if path == "/register":
|
||||
ip = self.client_address[0]
|
||||
if not _is_private(ip): # block tunnel-originated abuse
|
||||
self.send_error(403); return
|
||||
try:
|
||||
body = json.loads(self._body() or b"{}")
|
||||
except Exception:
|
||||
self.send_error(400); return
|
||||
have = body.get("have", []) or []
|
||||
C.box_state_save({"ip": ip, "id": body.get("id"),
|
||||
"last_seen": self.date_time_string(), "have": have})
|
||||
delta = C.compute_delta(C.staging_files(), have)
|
||||
print(f"register {ip} id={body.get('id')} have={len(have)} push={len(delta)}", flush=True)
|
||||
started = C.try_push(ip, delta)
|
||||
self._json({"queued": len(delta), "started": started}); return
|
||||
if not self._auth_ok():
|
||||
return
|
||||
if path == "/api/feeds":
|
||||
try:
|
||||
b = json.loads(self._body() or b"{}")
|
||||
C.add_feed(b["key"], b["name"], b["rss"], b.get("n", 2))
|
||||
except Exception as e:
|
||||
self._json({"error": str(e)}, 400); return
|
||||
self._json(C.load_feeds()); return
|
||||
if path == "/api/select":
|
||||
try:
|
||||
b = json.loads(self._body() or b"{}")
|
||||
key = b["feed"]; episodes = b["episodes"]
|
||||
except Exception as e:
|
||||
self._json({"error": str(e)}, 400); return
|
||||
feed = next((f for f in C.load_feeds() if f["key"] == key), None)
|
||||
if not feed:
|
||||
self._json({"error": "unknown feed"}, 404); return
|
||||
def work():
|
||||
cover_bytes = None
|
||||
try:
|
||||
_, cov = C.fetch_episodes(feed["rss"], limit=1)
|
||||
if cov:
|
||||
cover_bytes = C.fetch(cov)
|
||||
except Exception:
|
||||
cover_bytes = None
|
||||
for ep in episodes:
|
||||
try:
|
||||
C.add_episode(key, feed["name"], cover_bytes, ep["url"], ep.get("title", ""))
|
||||
cover_bytes = None
|
||||
print("selected", C.episode_fname(key, ep["url"]), flush=True)
|
||||
except Exception as e:
|
||||
print("select fail", e, flush=True)
|
||||
threading.Thread(target=work, daemon=True).start()
|
||||
self._json({"queued": len(episodes)}, 202); return
|
||||
if path == "/api/upload":
|
||||
q = urllib.parse.parse_qs(self.path.split("?", 1)[1] if "?" in self.path else "")
|
||||
show = (q.get("show") or ["Perso"])[0]
|
||||
title = (q.get("title") or ["Audio"])[0]
|
||||
data = self._body()
|
||||
if not data:
|
||||
self._json({"error": "empty"}, 400); return
|
||||
try:
|
||||
fname = C.add_upload(show, data, title)
|
||||
except Exception as e:
|
||||
self._json({"error": str(e)}, 500); return
|
||||
self._json({"file": fname}); return
|
||||
if path == "/api/push":
|
||||
st = C.box_state_load()
|
||||
ip = st.get("ip")
|
||||
if not ip:
|
||||
self._json({"error": "box never registered"}, 409); return
|
||||
delta = C.compute_delta(C.staging_files(), st.get("have", []))
|
||||
started = C.try_push(ip, delta)
|
||||
self._json({"ip": ip, "delta": len(delta), "started": started}); return
|
||||
self.send_error(404)
|
||||
|
||||
# --- DELETE ---
|
||||
def do_DELETE(self):
|
||||
if not self._auth_ok():
|
||||
return
|
||||
path = self.path.split("?")[0]
|
||||
if path.startswith("/api/feeds/"):
|
||||
C.remove_feed(urllib.parse.unquote(path[len("/api/feeds/"):]))
|
||||
self._json({"ok": True}); return
|
||||
if path.startswith("/api/files/"):
|
||||
name = urllib.parse.unquote(path[len("/api/files/"):])
|
||||
try:
|
||||
C.remove_file(name)
|
||||
except ValueError:
|
||||
self.send_error(400); return
|
||||
self._json({"ok": True}); return
|
||||
self.send_error(404)
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"lisael-content on :{PORT} staging={C.STAGING} auth={'on' if AUTH else 'OFF'}", flush=True)
|
||||
ThreadingHTTPServer(("0.0.0.0", PORT), H).serve_forever()
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Sanity** — `cd tower && MERLIN_AUTH= python3 -c "import ast; ast.parse(open('lisael_server.py').read()); print('ok')"` (le module importe lisael_content — ok si même dossier).
|
||||
- [ ] **Step 3: Smoke local** (sans réseau) : lancer le serveur avec un BASE temporaire, vérifier 401 sans auth + 200 sur `/api/feeds` avec auth.
|
||||
|
||||
```bash
|
||||
cd tower
|
||||
LISAEL_BASE=$(mktemp -d) MERLIN_AUTH=u:p LISAEL_PORT=9999 python3 lisael_server.py &
|
||||
sleep 1
|
||||
curl -s -o /dev/null -w "%{http_code}\n" localhost:9999/api/feeds # attendu 401
|
||||
curl -s -u u:p localhost:9999/api/feeds | head -c 60; echo # attendu JSON feeds
|
||||
kill %1
|
||||
```
|
||||
|
||||
- [ ] **Step 4: Commit** — `git commit -am "feat(tower): Merlin admin API + basic auth"`
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Frontend `merlin_ui.html`
|
||||
|
||||
**Files:** Create: `tower/merlin_ui.html`
|
||||
|
||||
Page HTML unique, vanilla (aucun framework, aucune dépendance CDN). Exigences :
|
||||
- **En-tête** : titre « 🪄 Merlin Studio » + sous-titre « Lisael Box ».
|
||||
- **3 onglets** (boutons en haut, une seule section visible) : Podcasts / Fichiers / Box.
|
||||
- **Onglet Podcasts** :
|
||||
- `GET /api/feeds` → cartes par flux (nom + bouton « voir les épisodes » + « ✕ »).
|
||||
- Formulaire « + ajouter un flux » : champs nom, URL RSS, n (défaut 2) → `POST /api/feeds` → recharger.
|
||||
- Clic « voir » → `GET /api/episodes?feed=<key>` → liste : checkbox + titre + durée + badge « ✓ sur la box » si `staged`. Bouton « Mettre sur la box » → `POST /api/select {feed, episodes:[{url,title}]}` (épisodes cochés non déjà staged) → toast « N en cours… » + après 3 s, refresh de l'onglet Fichiers.
|
||||
- Retrait flux → `DELETE /api/feeds/<key>`.
|
||||
- **Onglet Fichiers** :
|
||||
- `GET /api/files` → tableau (nom, taille lisible Mo/Ko, type) + bouton 🗑 par ligne → `DELETE /api/files/<name>` → refresh.
|
||||
- Zone upload : `<input type=file accept="audio/*">` + champ « émission » (texte, défaut « Perso ») + champ titre → lire le fichier en ArrayBuffer → `POST /api/upload?show=<>&title=<>` avec le corps = octets bruts, `Content-Type: audio/mpeg` → toast + refresh.
|
||||
- **Onglet Box** :
|
||||
- `GET /api/box` → afficher IP, dernière annonce, nb fichiers possédés (`box.have.length`), nb en attente (`delta.length`).
|
||||
- Bouton « Pousser maintenant » → `POST /api/push` → toast (`started`/`delta`), désactivé si pas d'IP.
|
||||
- **Style** : CSS inline, palette douce enfantine (fond clair, accents violet/orange), police système arrondie (`font-family: "Comic Sans MS", "Trebuchet MS", system-ui`), gros boutons tactiles, **responsive** (max-width 700px centré, marges fluides) — utilisable au téléphone.
|
||||
- **Auth** : aucune logique JS — le navigateur gère le prompt Basic Auth (le serveur renvoie 401 + WWW-Authenticate). Les `fetch` utilisent `credentials: "same-origin"`.
|
||||
- **Robustesse** : afficher les erreurs API dans un bandeau (réseau RSS lent → spinner « chargement… » sur l'onglet Podcasts).
|
||||
|
||||
- [ ] **Step 1: Écrire `tower/merlin_ui.html`** selon les exigences ci-dessus (page complète autoportante).
|
||||
- [ ] **Step 2: Valider** : ouvrir le fichier localement (`python3 -m http.server` ou `file://`) — vérifier que la structure des onglets s'affiche et que le JS ne lève pas d'erreur console (les `fetch` échoueront sans backend, c'est attendu ; pas d'exception JS au chargement).
|
||||
- [ ] **Step 3: Commit** — `git add tower/merlin_ui.html && git commit -m "feat(tower): Merlin Studio web UI"`
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Déploiement `merlin.saillant.cc` + E2E (INLINE — infra)
|
||||
|
||||
**Non délégué** (SSH Tower + proxy electron-server + matériel).
|
||||
|
||||
- [ ] **Step 1: Déployer sur Tower**
|
||||
|
||||
```bash
|
||||
scp tower/lisael_content.py tower/lisael_refresh.py tower/lisael_server.py \
|
||||
tower/merlin_ui.html [email protected]:/home/clems/lisael-content/
|
||||
```
|
||||
|
||||
- [ ] **Step 2: Auth + restart**
|
||||
|
||||
```bash
|
||||
ssh [email protected] 'sudo mkdir -p /etc/systemd/system/lisael-content.service.d && \
|
||||
printf "[Service]\nEnvironment=MERLIN_AUTH=merlin:<MOTDEPASSE>\n" | \
|
||||
sudo tee /etc/systemd/system/lisael-content.service.d/auth.conf >/dev/null && \
|
||||
sudo systemctl daemon-reload && sudo systemctl restart lisael-content.service && \
|
||||
systemctl is-active lisael-content.service'
|
||||
# vérifier l'API
|
||||
ssh [email protected] 'curl -s -o /dev/null -w "%{http_code}\n" localhost:9555/api/feeds; \
|
||||
curl -s -u merlin:<MOTDEPASSE> localhost:9555/api/feeds | head -c 80; echo'
|
||||
```
|
||||
Attendu : 401 sans auth, JSON feeds avec auth. (Générer un mot de passe fort, le donner à l'utilisateur.)
|
||||
|
||||
- [ ] **Step 3: Router `merlin.saillant.cc` → Tower:9555**
|
||||
Inspecter le proxy sur electron-server (`/home/electron/edge/docker-compose.yml` Traefik, ou tunnel cloudflared `reference_cloudflared_tunnel_saillant`). Ajouter une route/ingress `merlin.saillant.cc` → `http://100.78.6.122:9555`. Recharger le proxy. Vérifier la résolution DNS + 401 public.
|
||||
|
||||
```bash
|
||||
curl -s -o /dev/null -w "%{http_code}\n" https://merlin.saillant.cc/ # attendu 401
|
||||
curl -s -u merlin:<MOTDEPASSE> https://merlin.saillant.cc/api/feeds | head -c 80
|
||||
```
|
||||
|
||||
- [ ] **Step 4: E2E**
|
||||
- `GET /api/episodes?feed=oli` (avec auth) → renvoie des épisodes.
|
||||
- `POST /api/select` 1 épisode → après ~1 min, `GET /api/files` montre le nouveau `oli_*_a48.mp3` + manifeste à jour.
|
||||
- `POST /api/upload` un petit MP3 → apparaît dans `/api/files`.
|
||||
- `DELETE /api/files/<name>` → disparaît.
|
||||
- `GET /api/box` → état ; `POST /api/push` → push vers la box (si allumée), vérifier `push done` dans `journalctl -u lisael-content.service`.
|
||||
|
||||
- [ ] **Step 5: Mémoire** — consigner merlin.saillant.cc dans `project_lisael_box_hw_bringup_2026_06_14.md`.
|
||||
@@ -0,0 +1,125 @@
|
||||
# Merlin Studio — design (gestion de contenu Lisael Box)
|
||||
|
||||
**Date:** 2026-06-15
|
||||
**Status:** approuvé (verbal)
|
||||
|
||||
## Objectif
|
||||
|
||||
Une page web **`merlin.saillant.cc`** hébergée sur Tower pour gérer le contenu
|
||||
de la Lisael Box : **choisir les épisodes de podcasts** à mettre sur la box
|
||||
(curation parentale, épisode par épisode), **gérer les fichiers** du staging
|
||||
(voir / supprimer / uploader de l'audio perso), et **piloter la box** (état +
|
||||
push manuel). Complète l'auto-push hebdomadaire de la Spec A (qui reste
|
||||
automatique) par un contrôle manuel.
|
||||
|
||||
## Architecture
|
||||
|
||||
Tout vit sur **Tower** (`[email protected]`, tailnet `100.78.6.122`), là où
|
||||
sont déjà le staging, le push et l'auto-register :
|
||||
|
||||
```
|
||||
Navigateur ──HTTPS──► proxy electron-server ──tailnet──► Tower :9555
|
||||
merlin.saillant.cc (cloudflared/Traefik) lisael_server.py
|
||||
+ auth ├─ GET / UI
|
||||
├─ /api/* admin
|
||||
└─ POST /register box (LAN)
|
||||
|
||||
Box ──POST /register (LAN/tailnet, NON exposé au tunnel)──► Tower :9555
|
||||
Tower ──POST /put──► Box :8080 (push fichiers)
|
||||
```
|
||||
|
||||
Un **seul service** (`lisael_server.py`, déjà systemd `lisael-content.service`)
|
||||
sert l'API machine `/register` ET l'UI web + les APIs admin. Le navigateur tape
|
||||
`/` et `/api/*` ; la box tape `/register`.
|
||||
|
||||
## Composants
|
||||
|
||||
### `lisael_content.py` (nouveau module partagé)
|
||||
Toute la logique métier, importée par `lisael_refresh.py` (cron) ET
|
||||
`lisael_server.py` (web) :
|
||||
- **Config** : `load_feeds()` / `save_feeds(feeds)` → `feeds.json`
|
||||
(`[{key, name, rss, n}]`), seedé depuis les 4 flux actuels au premier appel.
|
||||
- **RSS** : `fetch(url)`, `cover_url(root)`, `clean(s)` (substitutions typo),
|
||||
`fetch_episodes(rss, limit=None)` → `[{url, title, duration, guid}]`.
|
||||
- **Transcode/pochette** : `episode_fname(key, url)` (`<key>_<sha1[:8]>_a48.mp3`),
|
||||
`transcode(raw_path, out_path)` (ffmpeg mono 48 kbps), `make_cover(img_bytes,
|
||||
key)` (160² → `.bin` RGB565 via LVGLImage).
|
||||
- **Staging** : `staging_files()`, `manifest_load()`, `manifest_save(m)`,
|
||||
`add_episode(key, show, rss_or_cover, url, title)` (download→transcode→staging
|
||||
+ upsert manifeste, idempotent par `fname`), `add_upload(show, src_bytes,
|
||||
title)` (transcode un MP3 perso → staging + manifeste, show « Perso »),
|
||||
`remove_file(name)` (supprime fichier + entrée manifeste + nettoie shows vides),
|
||||
`delete_file(name)`.
|
||||
- **Box** : `box_state_load()` / `box_state_save({ip,last_seen,have})`,
|
||||
`compute_delta(staging, have)`, `push_file(ip, name)`,
|
||||
`push_batch(ip, delta)` (thread de fond, garde anti-double-push par IP).
|
||||
|
||||
Écritures **atomiques** (fichier temp + `os.rename`) pour la sûreté ; plus de
|
||||
swap destructif du dossier entier.
|
||||
|
||||
### `lisael_refresh.py` (refactor — merge ADDITIF)
|
||||
Lit `feeds.json`, et pour chaque flux ajoute les **N derniers épisodes
|
||||
manquants** via `add_episode` (ne touche pas aux sélections manuelles ni aux
|
||||
uploads déjà présents), met à jour la pochette, reconstruit le manifeste en
|
||||
préservant l'existant. Cron hebdo inchangé (timer systemd dim. 04:00). Pas
|
||||
d'auto-prune en v1 (ménage manuel via l'onglet Fichiers).
|
||||
|
||||
### `lisael_server.py` (extension)
|
||||
- `GET /` → la page HTML (depuis `merlin_ui.html` lu sur disque).
|
||||
- `GET /api/feeds` ; `POST /api/feeds {key,name,rss,n}` ; `DELETE /api/feeds/<key>`.
|
||||
- `GET /api/episodes?feed=<key>` → épisodes du RSS + flag `staged` (fname déjà
|
||||
dans le staging).
|
||||
- `POST /api/select {feed, episodes:[{url,title}]}` → `add_episode` pour chacun
|
||||
(en thread de fond ; renvoie tout de suite, l'UI poll `/api/files`).
|
||||
- `GET /api/files` → `[{name,size,type}]` du staging.
|
||||
- `DELETE /api/files/<name>` → `remove_file`.
|
||||
- `POST /api/upload` (multipart : `show`, `file`) → `add_upload`.
|
||||
- `GET /api/box` → `box_state` + delta courant.
|
||||
- `POST /api/push` → `push_batch` vers la dernière IP connue.
|
||||
- `POST /register` (inchangé) → met à jour `box-state.json` + push async.
|
||||
- **Auth** : HTTP Basic Auth sur `/` et `/api/*` (identifiants depuis
|
||||
`$MERLIN_AUTH=user:pass`, fichier secret Tower) ; `/register` EXEMPT (machine,
|
||||
LAN). NB : la forward-auth Keycloak mutualisée est cassée (boucle, cf.
|
||||
`enfants.saillant.cc`) → Basic Auth in-app, portable et sans dépendance ; SSO
|
||||
dédié possible plus tard.
|
||||
|
||||
### `merlin_ui.html` (frontend, page unique)
|
||||
HTML/CSS/JS **vanilla** (aucun framework, cohérent avec le serveur stdlib).
|
||||
3 onglets :
|
||||
- **🎧 Podcasts** : liste des flux (`feeds.json`) avec ajout (nom + URL RSS) /
|
||||
retrait ; clic sur un flux → `GET /api/episodes` → liste épisodes (titre +
|
||||
durée + badge « déjà sur la box ») avec cases à cocher → bouton « Mettre sur
|
||||
la box » (`POST /api/select`).
|
||||
- **📂 Fichiers** : tableau du staging (nom, taille, type) + suppression ;
|
||||
zone d'upload MP3 (choix de l'émission cible).
|
||||
- **📦 Box** : dernière IP / date d'annonce / nombre de fichiers possédés +
|
||||
bouton « Pousser maintenant » (`POST /api/push`).
|
||||
Design enfantin léger (Fredoka-like, gros boutons), responsive (utilisable au
|
||||
téléphone).
|
||||
|
||||
## Modèle de données (sur Tower, `/home/clems/lisael-content/`)
|
||||
- `podcasts/` (staging) : `<key>_<hash>_a48.mp3`, `<key>.bin`, `manifest.json`.
|
||||
- `manifest.json` v2 (source de vérité du contenu) : `[{show, cover,
|
||||
episodes:[{file,title}]}]`.
|
||||
- `feeds.json` : `[{key, name, rss, n}]`.
|
||||
- `box-state.json` : `{ip, last_seen, have:[...]}`.
|
||||
|
||||
## Sécurité
|
||||
- Basic Auth devant `/` + `/api/*` ; `/register` ouvert sur le tailnet/LAN.
|
||||
- Le tunnel n'expose que `:9555` ; pas d'accès SSH/FS via la page.
|
||||
- Validation des noms de fichiers (pas de `..`, pas de `/` absolu) côté
|
||||
`/api/files` et upload.
|
||||
|
||||
## Déploiement
|
||||
- Copier `lisael_content.py`, `lisael_refresh.py`, `lisael_server.py`,
|
||||
`merlin_ui.html` dans `/home/clems/lisael-content/` ; `MERLIN_AUTH` via
|
||||
drop-in systemd ou `/home/clems/.secrets/` ; restart `lisael-content.service`.
|
||||
- Router `merlin.saillant.cc` → Tower `100.78.6.122:9555` via le proxy
|
||||
electron-server (cloudflared tunnel ou Traefik — à confirmer au déploiement,
|
||||
cf. `reference_cloudflared_tunnel_saillant`, `reference_git_saillant_cc_state`).
|
||||
- E2E : ouvrir la page, sélectionner un épisode → vérifier le staging, uploader
|
||||
un MP3, supprimer, « Pousser maintenant » vers la box.
|
||||
|
||||
## Hors périmètre v1
|
||||
- Auto-prune des orphelins ; SSO Keycloak dédié ; édition des pochettes ;
|
||||
gestion du contenu « Lire » (mots) ; multi-box.
|
||||
Reference in New Issue
Block a user