Merge pull request 'fix(plip): handset mic capture — ES8388 DLL + STT high-pass' (#164) from fix/plip-mic-stt-highpass into main

tools/kyutai-stt/kyutai_stt_server.py

@@ -119,6 +119,18 @@ def _run_inference(wav_path: str) -> str:
         # Mimi has reset_state() to clear its streaming KV-cache without reloading weights.
         _audio_tokenizer.reset_state()
 
+        # The Lm holds a PERSISTENT rotating KV-cache (`transformer_cache`) shared by
+        # every LmGen. LmGen.__init__ resets only its own gen_sequence/step_idx, NOT
+        # this cache — so across requests it keeps accumulating stream positions while
+        # each new LmGen restarts at step_idx=0. The positions drift out of alignment
+        # and the cache saturates, until the model emits only padding tokens (0/3) and
+        # every transcription comes back empty. Reset it per request, exactly as
+        # Lm.warmup() does at startup. (depformer_cache reset defensively too.)
+        for c in _lm_model.transformer_cache:
+            c.reset()
+        for c in _lm_model.depformer_cache:
+            c.reset()
+
         # LmGen is cheap to construct (no weight loading, just references the model).
         gen = models.LmGen(
             model=_lm_model,

tools/zacus-gateway/main.py

@@ -241,8 +241,10 @@ async def _chat_reply(http: httpx.AsyncClient, persona: str, history: list[dict]
     """
     messages = [{"role": "system", "content": persona + _PHONE_STYLE}] + history
     backends = [
-        # 60 s tolerates one cold model (re)load; a keep-warm task keeps it ~2 s.
+        # 70 s tolerates a ~47 s cold (re)load with margin; the keep-warm task
-        ("local", settings.local_chat_url, settings.local_chat_model, 60.0),
+        # keeps it ~2 s in normal operation. Total turn (STT ~10 s + this + say
+        # ~2 s) stays under the firmware's 90 s turn_client timeout.
+        ("local", settings.local_chat_url, settings.local_chat_model, 70.0),
         ("ailiance", settings.ailiance_tts_url, settings.ailiance_chat_model, 12.0),
     ]
     last_exc: Exception | None = None
@@ -285,8 +287,12 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
 
     async def _keep_llm_warm() -> None:
         """Ping the local LLM periodically so it never auto-unloads/cools between
-        calls — the first call after idle otherwise pays a ~40 s model reload.
+        calls — a cold (re)load takes ~47 s, long enough to blow the per-call
-        A tiny 1-token request loads (or keeps) the model resident."""
+        timeout and fall back. A tiny request keeps the model resident. Ping
+        every 4 min: well under the server's 1800 s idle-unload, with margin for
+        faster cooling under memory contention. On the very first ping (gateway
+        just started) the model loads cold — that's expected and only delays the
+        first call, which the 'un instant' filler covers."""
         while True:
             try:
                 await _chat_one(
@@ -296,7 +302,7 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
                 logging.info("LLM keep-warm: %s resident", settings.local_chat_model)
             except Exception as exc:  # noqa: BLE001 — best-effort
                 logging.warning("LLM keep-warm failed: %s", type(exc).__name__)
-            await asyncio.sleep(600)  # 10 min < auto-unload-idle (1800 s)
+            await asyncio.sleep(240)  # 4 min ≪ 1800 s idle-unload, margin for contention
 
     prewarm_task = asyncio.create_task(_prewarm_greetings())
     warm_task = asyncio.create_task(_keep_llm_warm())
@@ -1908,12 +1914,17 @@ def _build_wav(pcm: bytes, sr: int, ch: int, sw: int) -> bytes:
     return out.getvalue()
 
 
-def _normalize_wav_for_stt(wav_bytes: bytes, target_peak: float = 0.5,
+def _normalize_wav_for_stt(wav_bytes: bytes, target_peak: float = 0.7,
-                           max_gain: float = 25.0) -> bytes:
+                           max_gain: float = 40.0, hp_cutoff_hz: float = 110.0) -> bytes:
-    """Boost a quiet capture up to `target_peak` before STT. The PLIP handset
+    """Condition a PLIP handset capture for Kyutai STT: high-pass then normalise.
-    mic comes in very low (~1-4 % FS through the SLIC), too quiet for Kyutai to
+
-    transcribe — it hears only the noise floor and hallucinates. Peak-normalise
+    Two problems with the SLIC handset path: (1) it injects low-frequency rumble
-    in software (gain capped so pure silence isn't blown up into noise)."""
+    / DC drift that swamps the speech and makes Kyutai output nothing — a
+    high-pass (~110 Hz, box moving-average subtraction) removes it; (2) the mic
+    comes in very quiet (~2-5 % FS even at +24 dB PGA), so peak-normalise after
+    filtering (gain capped so silence isn't blown up). WITHOUT the high-pass the
+    exact same capture transcribes empty; WITH it, clean French — verified on the
+    bench."""
     if _np is None:
         return wav_bytes
     parsed = _wav_pcm(wav_bytes)
@@ -1923,12 +1934,17 @@ def _normalize_wav_for_stt(wav_bytes: bytes, target_peak: float = 0.5,
     if sw != 2 or not pcm:
         return wav_bytes
     arr = _np.frombuffer(pcm, dtype=_np.int16).astype(_np.float32)
+    # High-pass = signal minus its low-frequency moving average. Window length
+    # sets the cutoff (~sr/k Hz); k≈145 @16 kHz → ~110 Hz.
+    k = max(2, int(sr / max(hp_cutoff_hz, 1.0)))
+    if arr.size > k:
+        lp = _np.convolve(arr, _np.ones(k, dtype=_np.float32) / k, mode="same")
+        arr = arr - lp
     peak = float(_np.max(_np.abs(arr))) or 1.0
     gain = min((target_peak * 32767.0) / peak, max_gain)
-    if gain <= 1.05:  # already loud enough
-        return _build_wav(pcm, sr, ch, sw)
     arr = _np.clip(arr * gain, -32768, 32767).astype(_np.int16)
-    logging.info("STT normalise: peak %.1f%% FS → gain x%.1f", 100 * peak / 32768, gain)
+    logging.info("STT conditioning: high-pass %.0f Hz, peak %.1f%% FS → gain x%.1f",
+                 hp_cutoff_hz, 100 * peak / 32768, gain)
     return _build_wav(arr.tobytes(), sr, ch, sw)