electron-rare/le-mystere-professeur-zacus
1
0
Fork 0
Code Issues 1 Pull Requests 2 Actions 1 Packages Projects Releases Wiki Activity

fix(gateway): snappier LLM keep-warm + higher timeout

Browse Source 
- keep-warm interval 600s -> 240s (avoids ~47s cold-start fallback)
- local backend timeout 60s -> 70s (tolerates cold reload under 90s)
This commit is contained in:
clement
2026-06-16 10:32:37 +02:00
parent 1e15bba258
commit 8d90a25756
1 changed files with 11 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/zacus-gateway/main.py
+11 -5
View File
@@ -241,8 +241,10 @@ async def _chat_reply(http: httpx.AsyncClient, persona: str, history: list[dict]
""" """
messages = [{"role": "system", "content": persona + _PHONE_STYLE}] + history messages = [{"role": "system", "content": persona + _PHONE_STYLE}] + history
backends = [ backends = [
# 60 s tolerates one cold model (re)load; a keep-warm task keeps it ~2 s. # 70 s tolerates a ~47 s cold (re)load with margin; the keep-warm task
("local", settings.local_chat_url, settings.local_chat_model, 60.0), # keeps it ~2 s in normal operation. Total turn (STT ~10 s + this + say
# ~2 s) stays under the firmware's 90 s turn_client timeout.
("local", settings.local_chat_url, settings.local_chat_model, 70.0),
("ailiance", settings.ailiance_tts_url, settings.ailiance_chat_model, 12.0), ("ailiance", settings.ailiance_tts_url, settings.ailiance_chat_model, 12.0),
] ]
last_exc: Exception | None = None last_exc: Exception | None = None
@@ -285,8 +287,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
async def _keep_llm_warm() -> None: async def _keep_llm_warm() -> None:
"""Ping the local LLM periodically so it never auto-unloads/cools between """Ping the local LLM periodically so it never auto-unloads/cools between
calls — the first call after idle otherwise pays a ~40 s model reload. calls — a cold (re)load takes ~47 s, long enough to blow the per-call
A tiny 1-token request loads (or keeps) the model resident.""" timeout and fall back. A tiny request keeps the model resident. Ping
every 4 min: well under the server's 1800 s idle-unload, with margin for
faster cooling under memory contention. On the very first ping (gateway
just started) the model loads cold — that's expected and only delays the
first call, which the 'un instant' filler covers."""
while True: while True:
try: try:
await _chat_one( await _chat_one(
@@ -296,7 +302,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
logging.info("LLM keep-warm: %s resident", settings.local_chat_model) logging.info("LLM keep-warm: %s resident", settings.local_chat_model)
except Exception as exc: # noqa: BLE001 — best-effort except Exception as exc: # noqa: BLE001 — best-effort
logging.warning("LLM keep-warm failed: %s", type(exc).__name__) logging.warning("LLM keep-warm failed: %s", type(exc).__name__)
await asyncio.sleep(600) # 10 min < auto-unload-idle (1800 s) await asyncio.sleep(240) # 4 min ≪ 1800 s idle-unload, margin for contention
prewarm_task = asyncio.create_task(_prewarm_greetings()) prewarm_task = asyncio.create_task(_prewarm_greetings())
warm_task = asyncio.create_task(_keep_llm_warm()) warm_task = asyncio.create_task(_keep_llm_warm())