From bc5363cf2cef079bf55c18e8d0bbbe768639762f Mon Sep 17 00:00:00 2001 From: clement Date: Tue, 16 Jun 2026 09:35:00 +0200 Subject: [PATCH 1/7] fix(gateway): STT high-pass + gain for SLIC mic SLIC handset injects low-frequency rumble/DC drift that swamps speech and causes Kyutai STT to output nothing. Apply a box moving-average high-pass (~110 Hz) before peak-normalisation. Also raise gain cap to x40 and target_peak to 0.7 (mic is still very quiet post-ES8388 fix). Verified: same capture transcribes empty without HP, clean French with. --- tools/zacus-gateway/main.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/tools/zacus-gateway/main.py b/tools/zacus-gateway/main.py index 7b6f547..5e07ad4 100644 --- a/tools/zacus-gateway/main.py +++ b/tools/zacus-gateway/main.py @@ -1908,12 +1908,17 @@ def _build_wav(pcm: bytes, sr: int, ch: int, sw: int) -> bytes: return out.getvalue() -def _normalize_wav_for_stt(wav_bytes: bytes, target_peak: float = 0.5, - max_gain: float = 25.0) -> bytes: - """Boost a quiet capture up to `target_peak` before STT. The PLIP handset - mic comes in very low (~1-4 % FS through the SLIC), too quiet for Kyutai to - transcribe — it hears only the noise floor and hallucinates. Peak-normalise - in software (gain capped so pure silence isn't blown up into noise).""" +def _normalize_wav_for_stt(wav_bytes: bytes, target_peak: float = 0.7, + max_gain: float = 40.0, hp_cutoff_hz: float = 110.0) -> bytes: + """Condition a PLIP handset capture for Kyutai STT: high-pass then normalise. + + Two problems with the SLIC handset path: (1) it injects low-frequency rumble + / DC drift that swamps the speech and makes Kyutai output nothing — a + high-pass (~110 Hz, box moving-average subtraction) removes it; (2) the mic + comes in very quiet (~2-5 % FS even at +24 dB PGA), so peak-normalise after + filtering (gain capped so silence isn't blown up). WITHOUT the high-pass the + exact same capture transcribes empty; WITH it, clean French — verified on the + bench.""" if _np is None: return wav_bytes parsed = _wav_pcm(wav_bytes) @@ -1923,12 +1928,17 @@ def _normalize_wav_for_stt(wav_bytes: bytes, target_peak: float = 0.5, if sw != 2 or not pcm: return wav_bytes arr = _np.frombuffer(pcm, dtype=_np.int16).astype(_np.float32) + # High-pass = signal minus its low-frequency moving average. Window length + # sets the cutoff (~sr/k Hz); k≈145 @16 kHz → ~110 Hz. + k = max(2, int(sr / max(hp_cutoff_hz, 1.0))) + if arr.size > k: + lp = _np.convolve(arr, _np.ones(k, dtype=_np.float32) / k, mode="same") + arr = arr - lp peak = float(_np.max(_np.abs(arr))) or 1.0 gain = min((target_peak * 32767.0) / peak, max_gain) - if gain <= 1.05: # already loud enough - return _build_wav(pcm, sr, ch, sw) arr = _np.clip(arr * gain, -32768, 32767).astype(_np.int16) - logging.info("STT normalise: peak %.1f%% FS → gain x%.1f", 100 * peak / 32768, gain) + logging.info("STT conditioning: high-pass %.0f Hz, peak %.1f%% FS → gain x%.1f", + hp_cutoff_hz, 100 * peak / 32768, gain) return _build_wav(arr.tobytes(), sr, ch, sw) From 1e15bba2582098f07fe050efdf338eac57e210be Mon Sep 17 00:00:00 2001 From: clement Date: Tue, 16 Jun 2026 09:35:20 +0200 Subject: [PATCH 2/7] =?UTF-8?q?chore:=20bump=20ESP32=5FZACUS=20=E2=80=94?= =?UTF-8?q?=20ES8388=20DLL=20ADC=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ESP32_ZACUS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ESP32_ZACUS b/ESP32_ZACUS index ebcfb01..4a6fc43 160000 --- a/ESP32_ZACUS +++ b/ESP32_ZACUS @@ -1 +1 @@ -Subproject commit ebcfb011d3c67149ade5c7573048c6752a3b6e6b +Subproject commit 4a6fc435a73be5ff5c1e8c74681e6a613fa5ac48 From 8d90a25756b25a2f99c43446988c5538875c77d8 Mon Sep 17 00:00:00 2001 From: clement Date: Tue, 16 Jun 2026 10:32:37 +0200 Subject: [PATCH 3/7] fix(gateway): snappier LLM keep-warm + higher timeout - keep-warm interval 600s -> 240s (avoids ~47s cold-start fallback) - local backend timeout 60s -> 70s (tolerates cold reload under 90s) --- tools/zacus-gateway/main.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/tools/zacus-gateway/main.py b/tools/zacus-gateway/main.py index 5e07ad4..b16d47a 100644 --- a/tools/zacus-gateway/main.py +++ b/tools/zacus-gateway/main.py @@ -241,8 +241,10 @@ async def _chat_reply(http: httpx.AsyncClient, persona: str, history: list[dict] """ messages = [{"role": "system", "content": persona + _PHONE_STYLE}] + history backends = [ - # 60 s tolerates one cold model (re)load; a keep-warm task keeps it ~2 s. - ("local", settings.local_chat_url, settings.local_chat_model, 60.0), + # 70 s tolerates a ~47 s cold (re)load with margin; the keep-warm task + # keeps it ~2 s in normal operation. Total turn (STT ~10 s + this + say + # ~2 s) stays under the firmware's 90 s turn_client timeout. + ("local", settings.local_chat_url, settings.local_chat_model, 70.0), ("ailiance", settings.ailiance_tts_url, settings.ailiance_chat_model, 12.0), ] last_exc: Exception | None = None @@ -285,8 +287,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: async def _keep_llm_warm() -> None: """Ping the local LLM periodically so it never auto-unloads/cools between - calls — the first call after idle otherwise pays a ~40 s model reload. - A tiny 1-token request loads (or keeps) the model resident.""" + calls — a cold (re)load takes ~47 s, long enough to blow the per-call + timeout and fall back. A tiny request keeps the model resident. Ping + every 4 min: well under the server's 1800 s idle-unload, with margin for + faster cooling under memory contention. On the very first ping (gateway + just started) the model loads cold — that's expected and only delays the + first call, which the 'un instant' filler covers.""" while True: try: await _chat_one( @@ -296,7 +302,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: logging.info("LLM keep-warm: %s resident", settings.local_chat_model) except Exception as exc: # noqa: BLE001 — best-effort logging.warning("LLM keep-warm failed: %s", type(exc).__name__) - await asyncio.sleep(600) # 10 min < auto-unload-idle (1800 s) + await asyncio.sleep(240) # 4 min ≪ 1800 s idle-unload, margin for contention prewarm_task = asyncio.create_task(_prewarm_greetings()) warm_task = asyncio.create_task(_keep_llm_warm()) From f8e576ba5a81068ef555a7fc7c49489b1cfb6ec2 Mon Sep 17 00:00:00 2001 From: clement Date: Tue, 16 Jun 2026 10:32:43 +0200 Subject: [PATCH 4/7] =?UTF-8?q?chore:=20bump=20ESP32=5FZACUS=20=E2=80=94?= =?UTF-8?q?=20incoming=20mode=20+=20volume?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ESP32_ZACUS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ESP32_ZACUS b/ESP32_ZACUS index 4a6fc43..ccc84da 160000 --- a/ESP32_ZACUS +++ b/ESP32_ZACUS @@ -1 +1 @@ -Subproject commit 4a6fc435a73be5ff5c1e8c74681e6a613fa5ac48 +Subproject commit ccc84da7851564fe561bfc95cd6202ff541c668d From f703611388caa824062855df4af9add18d516a1e Mon Sep 17 00:00:00 2001 From: clement Date: Tue, 16 Jun 2026 19:19:35 +0200 Subject: [PATCH 5/7] =?UTF-8?q?chore:=20bump=20ESP32=5FZACUS=20=E2=80=94?= =?UTF-8?q?=20status=20+=20capture=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ESP32_ZACUS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ESP32_ZACUS b/ESP32_ZACUS index ccc84da..99d79ef 160000 --- a/ESP32_ZACUS +++ b/ESP32_ZACUS @@ -1 +1 @@ -Subproject commit ccc84da7851564fe561bfc95cd6202ff541c668d +Subproject commit 99d79efcddf5dfefa50cd01966a084d83d24f990 From fb71e9ceeb79d79719fa6d6b4dfdf11cd8b9ef4c Mon Sep 17 00:00:00 2001 From: clement Date: Wed, 17 Jun 2026 08:26:05 +0200 Subject: [PATCH 6/7] fix(stt): reset LM KV-cache per request The Lm holds a persistent rotating transformer_cache shared by every LmGen. LmGen only resets its own gen_sequence/step_idx, so across requests the cache kept accumulating stream positions until it saturated and the model emitted only padding tokens, returning empty transcriptions ('') for every request. Reset transformer_cache (and depformer_cache) per request, mirroring Lm.warmup(). Validated: 40 consecutive requests of a known-good sample, 0 empty. --- tools/kyutai-stt/kyutai_stt_server.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/kyutai-stt/kyutai_stt_server.py b/tools/kyutai-stt/kyutai_stt_server.py index e736425..c39624c 100644 --- a/tools/kyutai-stt/kyutai_stt_server.py +++ b/tools/kyutai-stt/kyutai_stt_server.py @@ -119,6 +119,18 @@ def _run_inference(wav_path: str) -> str: # Mimi has reset_state() to clear its streaming KV-cache without reloading weights. _audio_tokenizer.reset_state() + # The Lm holds a PERSISTENT rotating KV-cache (`transformer_cache`) shared by + # every LmGen. LmGen.__init__ resets only its own gen_sequence/step_idx, NOT + # this cache — so across requests it keeps accumulating stream positions while + # each new LmGen restarts at step_idx=0. The positions drift out of alignment + # and the cache saturates, until the model emits only padding tokens (0/3) and + # every transcription comes back empty. Reset it per request, exactly as + # Lm.warmup() does at startup. (depformer_cache reset defensively too.) + for c in _lm_model.transformer_cache: + c.reset() + for c in _lm_model.depformer_cache: + c.reset() + # LmGen is cheap to construct (no weight loading, just references the model). gen = models.LmGen( model=_lm_model, From af7c02d1a75eff4cd4d343634cde86af6acecf7f Mon Sep 17 00:00:00 2001 From: clement Date: Wed, 17 Jun 2026 09:06:19 +0200 Subject: [PATCH 7/7] chore: bump ESP32_ZACUS (no hook forcing) Pull in ESP32_ZACUS fix(plip) 8c076d8: gate audio playback on the real SHK line and remove all hook forcing (s_hook_override, phone_force_offhook, /debug/offhook). Fixes the silent incoming-call greeting caused by the two off-hook flags desyncing during ring. --- ESP32_ZACUS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ESP32_ZACUS b/ESP32_ZACUS index 99d79ef..8c076d8 160000 --- a/ESP32_ZACUS +++ b/ESP32_ZACUS @@ -1 +1 @@ -Subproject commit 99d79efcddf5dfefa50cd01966a084d83d24f990 +Subproject commit 8c076d81d6efe4e7ddb6e21a4934e602275d19b0