feat(tower): cap shows at latest N per feed
The additive refresh accumulated episodes without bound (Oli reached 13). Tag each manifest episode with a source (auto/pick/upload); the refresh now prunes "auto" episodes that aged out of the feed's latest N while keeping hand-picked and uploaded ones. Feeds default to N=5.
This commit is contained in:
+35
-7
@@ -139,7 +139,9 @@ def manifest_load():
|
||||
def manifest_save(m):
|
||||
_atomic_write_json(os.path.join(STAGING, "manifest.json"), m)
|
||||
|
||||
def _manifest_upsert(m, show, cover, file, title):
|
||||
def _manifest_upsert(m, show, cover, file, title, src="auto"):
|
||||
# src: "auto" (weekly refresh), "pick" (hand-selected), "upload" (custom file).
|
||||
# Only "auto" episodes are eligible for pruning by the cap.
|
||||
for s in m:
|
||||
if s["show"] == show:
|
||||
if cover and not s.get("cover"):
|
||||
@@ -147,14 +149,17 @@ def _manifest_upsert(m, show, cover, file, title):
|
||||
for e in s["episodes"]:
|
||||
if e["file"] == file:
|
||||
e["title"] = title
|
||||
if src != "auto": # a manual pick/upload wins over auto
|
||||
e["src"] = src
|
||||
return
|
||||
s["episodes"].append({"file": file, "title": title})
|
||||
s["episodes"].append({"file": file, "title": title, "src": src})
|
||||
return
|
||||
m.append({"show": show, "cover": cover or "", "episodes": [{"file": file, "title": title}]})
|
||||
m.append({"show": show, "cover": cover or "",
|
||||
"episodes": [{"file": file, "title": title, "src": src}]})
|
||||
|
||||
def add_episode(key, show, cover_bytes, url, title):
|
||||
def add_episode(key, show, cover_bytes, url, title, src="auto"):
|
||||
"""Download + transcode an episode into STAGING + upsert manifest. Idempotent by fname.
|
||||
cover_bytes: raw image bytes for the show cover, or None."""
|
||||
cover_bytes: raw image bytes for the show cover, or None. src: auto|pick."""
|
||||
os.makedirs(STAGING, exist_ok=True)
|
||||
fname = episode_fname(key, url)
|
||||
out = os.path.join(STAGING, fname)
|
||||
@@ -167,7 +172,7 @@ def add_episode(key, show, cover_bytes, url, title):
|
||||
os.replace(tmp_out, out)
|
||||
cover = make_cover(cover_bytes, key) if cover_bytes else ""
|
||||
m = manifest_load()
|
||||
_manifest_upsert(m, show, cover, fname, clean(title))
|
||||
_manifest_upsert(m, show, cover, fname, clean(title), src)
|
||||
manifest_save(m)
|
||||
return fname
|
||||
|
||||
@@ -183,10 +188,33 @@ def add_upload(show, src_bytes, title, key="perso"):
|
||||
raise RuntimeError("transcode failed")
|
||||
os.replace(tmp_out, out)
|
||||
m = manifest_load()
|
||||
_manifest_upsert(m, show, "", fname, clean(title))
|
||||
_manifest_upsert(m, show, "", fname, clean(title), "upload")
|
||||
manifest_save(m)
|
||||
return fname
|
||||
|
||||
def prune_show(show, keep_files):
|
||||
"""Cap a feed show: drop its 'auto' episodes whose file is not in keep_files
|
||||
(the current latest-N from the feed). 'pick'/'upload' episodes are always kept.
|
||||
Deletes the staged file too. Returns the number removed."""
|
||||
m = manifest_load()
|
||||
removed = 0
|
||||
for s in m:
|
||||
if s["show"] != show:
|
||||
continue
|
||||
kept = []
|
||||
for e in s["episodes"]:
|
||||
if e.get("src", "auto") != "auto" or e["file"] in keep_files:
|
||||
kept.append(e)
|
||||
else:
|
||||
p = os.path.join(STAGING, e["file"])
|
||||
if os.path.exists(p):
|
||||
os.remove(p)
|
||||
removed += 1
|
||||
s["episodes"] = kept
|
||||
m = [s for s in m if s["episodes"]]
|
||||
manifest_save(m)
|
||||
return removed
|
||||
|
||||
def remove_file(name):
|
||||
if "/" in name or ".." in name or name.startswith("."):
|
||||
raise ValueError("bad name")
|
||||
|
||||
+13
-7
@@ -1,12 +1,13 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Weekly refresh: for each configured feed, add the latest N missing episodes to
|
||||
the staging set. ADDITIVE — never wipes manual selections or uploads."""
|
||||
"""Weekly refresh: for each configured feed, keep the latest N episodes staged.
|
||||
Adds the new ones (src=auto) and prunes auto episodes that aged out of the feed's
|
||||
latest N. Hand-picked ('pick') and uploaded ('upload') episodes are never touched."""
|
||||
import lisael_content as C
|
||||
|
||||
def main():
|
||||
added = 0
|
||||
added = removed = 0
|
||||
for fd in C.load_feeds():
|
||||
key, show, rss, n = fd["key"], fd["name"], fd["rss"], int(fd.get("n", 2))
|
||||
key, show, rss, n = fd["key"], fd["name"], fd["rss"], int(fd.get("n", 5))
|
||||
try:
|
||||
eps, cov = C.fetch_episodes(rss, limit=n)
|
||||
except Exception as e:
|
||||
@@ -18,17 +19,22 @@ def main():
|
||||
except Exception:
|
||||
cover_bytes = None
|
||||
staged = set(C.staging_files())
|
||||
want = set()
|
||||
for ep in eps:
|
||||
if C.episode_fname(key, ep["url"]) in staged:
|
||||
fn = C.episode_fname(key, ep["url"])
|
||||
want.add(fn)
|
||||
if fn in staged:
|
||||
continue
|
||||
try:
|
||||
C.add_episode(key, show, cover_bytes, ep["url"], ep["title"])
|
||||
cover_bytes = None # cover only needs writing once per show
|
||||
added += 1
|
||||
print("added", C.episode_fname(key, ep["url"]), flush=True)
|
||||
print("added", fn, flush=True)
|
||||
except Exception as e:
|
||||
print("add fail", key, e, flush=True)
|
||||
print(f"refresh done: +{added} episodes, {len(C.staging_files())} files", flush=True)
|
||||
# Cap: drop auto episodes that are no longer in the feed's latest N.
|
||||
removed += C.prune_show(show, want)
|
||||
print(f"refresh done: +{added} -{removed}, {len(C.staging_files())} files", flush=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -138,7 +138,7 @@ class H(BaseHTTPRequestHandler):
|
||||
cover_bytes = None
|
||||
for ep in episodes:
|
||||
try:
|
||||
C.add_episode(key, feed["name"], cover_bytes, ep["url"], ep.get("title", ""))
|
||||
C.add_episode(key, feed["name"], cover_bytes, ep["url"], ep.get("title", ""), src="pick")
|
||||
cover_bytes = None
|
||||
print("selected", C.episode_fname(key, ep["url"]), flush=True)
|
||||
except Exception as e:
|
||||
|
||||
@@ -41,6 +41,25 @@ def test_bad_name_rejected():
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def test_prune_show_caps_auto_keeps_manual():
|
||||
os.makedirs(C.STAGING, exist_ok=True)
|
||||
m = []
|
||||
# 2 auto (one stays, one ages out) + 1 hand-picked old + 1 upload
|
||||
C._manifest_upsert(m, "Sh", "sh.bin", "sh_new_a48.mp3", "new", "auto")
|
||||
C._manifest_upsert(m, "Sh", "", "sh_old_a48.mp3", "old", "auto")
|
||||
C._manifest_upsert(m, "Sh", "", "sh_pick_a48.mp3", "picked", "pick")
|
||||
C._manifest_upsert(m, "Sh", "", "sh_up_a48.mp3", "uploaded", "upload")
|
||||
C.manifest_save(m)
|
||||
for f in ("sh_new_a48.mp3", "sh_old_a48.mp3", "sh_pick_a48.mp3", "sh_up_a48.mp3"):
|
||||
open(os.path.join(C.STAGING, f), "wb").close()
|
||||
# keep only the new auto episode; old auto should be pruned, manual kept
|
||||
removed = C.prune_show("Sh", {"sh_new_a48.mp3"})
|
||||
assert removed == 1
|
||||
files = {e["file"] for e in C.manifest_load()[0]["episodes"]}
|
||||
assert files == {"sh_new_a48.mp3", "sh_pick_a48.mp3", "sh_up_a48.mp3"}
|
||||
assert not os.path.exists(os.path.join(C.STAGING, "sh_old_a48.mp3"))
|
||||
assert os.path.exists(os.path.join(C.STAGING, "sh_pick_a48.mp3"))
|
||||
|
||||
if __name__ == "__main__":
|
||||
for name, fn in sorted(globals().items()):
|
||||
if name.startswith("test_"):
|
||||
|
||||
Reference in New Issue
Block a user