From 8d90a25756b25a2f99c43446988c5538875c77d8 Mon Sep 17 00:00:00 2001 From: clement Date: Tue, 16 Jun 2026 10:32:37 +0200 Subject: [PATCH] fix(gateway): snappier LLM keep-warm + higher timeout - keep-warm interval 600s -> 240s (avoids ~47s cold-start fallback) - local backend timeout 60s -> 70s (tolerates cold reload under 90s) --- tools/zacus-gateway/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/zacus-gateway/main.py b/tools/zacus-gateway/main.py index 5e07ad4..b16d47a 100644 --- a/tools/zacus-gateway/main.py +++ b/tools/zacus-gateway/main.py @@ -241,8 +241,10 @@ async def _chat_reply(http: httpx.AsyncClient, persona: str, history: list[dict] """ messages = [{"role": "system", "content": persona + _PHONE_STYLE}] + history backends = [ - # 60 s tolerates one cold model (re)load; a keep-warm task keeps it ~2 s. - ("local", settings.local_chat_url, settings.local_chat_model, 60.0), + # 70 s tolerates a ~47 s cold (re)load with margin; the keep-warm task + # keeps it ~2 s in normal operation. Total turn (STT ~10 s + this + say + # ~2 s) stays under the firmware's 90 s turn_client timeout. + ("local", settings.local_chat_url, settings.local_chat_model, 70.0), ("ailiance", settings.ailiance_tts_url, settings.ailiance_chat_model, 12.0), ] last_exc: Exception | None = None @@ -285,8 +287,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: async def _keep_llm_warm() -> None: """Ping the local LLM periodically so it never auto-unloads/cools between - calls — the first call after idle otherwise pays a ~40 s model reload. - A tiny 1-token request loads (or keeps) the model resident.""" + calls — a cold (re)load takes ~47 s, long enough to blow the per-call + timeout and fall back. A tiny request keeps the model resident. Ping + every 4 min: well under the server's 1800 s idle-unload, with margin for + faster cooling under memory contention. On the very first ping (gateway + just started) the model loads cold — that's expected and only delays the + first call, which the 'un instant' filler covers.""" while True: try: await _chat_one( @@ -296,7 +302,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: logging.info("LLM keep-warm: %s resident", settings.local_chat_model) except Exception as exc: # noqa: BLE001 — best-effort logging.warning("LLM keep-warm failed: %s", type(exc).__name__) - await asyncio.sleep(600) # 10 min < auto-unload-idle (1800 s) + await asyncio.sleep(240) # 4 min ≪ 1800 s idle-unload, margin for contention prewarm_task = asyncio.create_task(_prewarm_greetings()) warm_task = asyncio.create_task(_keep_llm_warm())