fix(gateway): snappier LLM keep-warm + higher timeout
- keep-warm interval 600s -> 240s (avoids ~47s cold-start fallback) - local backend timeout 60s -> 70s (tolerates cold reload under 90s)
This commit is contained in:
@@ -241,8 +241,10 @@ async def _chat_reply(http: httpx.AsyncClient, persona: str, history: list[dict]
|
||||
"""
|
||||
messages = [{"role": "system", "content": persona + _PHONE_STYLE}] + history
|
||||
backends = [
|
||||
# 60 s tolerates one cold model (re)load; a keep-warm task keeps it ~2 s.
|
||||
("local", settings.local_chat_url, settings.local_chat_model, 60.0),
|
||||
# 70 s tolerates a ~47 s cold (re)load with margin; the keep-warm task
|
||||
# keeps it ~2 s in normal operation. Total turn (STT ~10 s + this + say
|
||||
# ~2 s) stays under the firmware's 90 s turn_client timeout.
|
||||
("local", settings.local_chat_url, settings.local_chat_model, 70.0),
|
||||
("ailiance", settings.ailiance_tts_url, settings.ailiance_chat_model, 12.0),
|
||||
]
|
||||
last_exc: Exception | None = None
|
||||
@@ -285,8 +287,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
|
||||
async def _keep_llm_warm() -> None:
|
||||
"""Ping the local LLM periodically so it never auto-unloads/cools between
|
||||
calls — the first call after idle otherwise pays a ~40 s model reload.
|
||||
A tiny 1-token request loads (or keeps) the model resident."""
|
||||
calls — a cold (re)load takes ~47 s, long enough to blow the per-call
|
||||
timeout and fall back. A tiny request keeps the model resident. Ping
|
||||
every 4 min: well under the server's 1800 s idle-unload, with margin for
|
||||
faster cooling under memory contention. On the very first ping (gateway
|
||||
just started) the model loads cold — that's expected and only delays the
|
||||
first call, which the 'un instant' filler covers."""
|
||||
while True:
|
||||
try:
|
||||
await _chat_one(
|
||||
@@ -296,7 +302,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
logging.info("LLM keep-warm: %s resident", settings.local_chat_model)
|
||||
except Exception as exc: # noqa: BLE001 — best-effort
|
||||
logging.warning("LLM keep-warm failed: %s", type(exc).__name__)
|
||||
await asyncio.sleep(600) # 10 min < auto-unload-idle (1800 s)
|
||||
await asyncio.sleep(240) # 4 min ≪ 1800 s idle-unload, margin for contention
|
||||
|
||||
prewarm_task = asyncio.create_task(_prewarm_greetings())
|
||||
warm_task = asyncio.create_task(_keep_llm_warm())
|
||||
|
||||
Reference in New Issue
Block a user