feat(voice): Whisper STT (robust to handset voice)
Validate Zacus refactor / validate (push) Failing after 12m37s
Repo State / repo-state (push) Failing after 13m3s
Validate Zacus refactor / validate (pull_request) Failing after 13m17s
Repo State / repo-state (pull_request) Failing after 13m35s

Switch the voice-loop STT from Kyutai to whisper.cpp large-v3-turbo
(subprocess, high-pass conditioned): it decodes the weak/noisy SLIC
handset captures Kyutai returned '' for. Filter whisper's silence/noise
hallucinations (sous-titrage, radio-canada, tipeurs, '...') so the NPC
never replies to a phantom line. Bump ESP32_ZACUS to 54022ed (VAD
recalibration for the quiet voice + bell stop on pickup).
This commit was merged in pull request #167.
This commit is contained in:
clement
2026-06-18 00:08:59 +02:00
parent 643c3852de
commit 73fa3650a1
2 changed files with 48 additions and 2 deletions
+47 -1
View File
@@ -2004,6 +2004,52 @@ async def _transcribe_kyutai(http: httpx.AsyncClient, audio_wav: bytes) -> str:
return (resp.json().get("text") or "").strip()
# whisper.cpp (large-v3-turbo, CoreML) — far more robust to the weak/noisy SLIC
# handset captures than Kyutai's streaming STT (decodes voice Kyutai returns ''
# for). Used as the primary STT for the voice loop.
_WHISPER_BIN = "/Users/electron/whisperx-server/whisper.cpp/build/bin/whisper-cli"
_WHISPER_MODEL = "/Users/electron/whisperx-server/models/ggml-large-v3-turbo-q5_0.bin"
_WHISPER_NOISE = re.compile(r"\[(?:_?BLANK_AUDIO_?|BLANK_AUDIO|silence|musique|music)\]|\(\s*\)", re.I)
# whisper hallucinates these training-data artefacts on silence/noise — treat as empty.
_WHISPER_HALLUC = re.compile(
r"sous-titr|société radio-canada|radio-canada|merci d'avoir regard|"
r"amara\.org|abonnez-vous|tipeur|souscripteur|merci à tous|"
r"n'oubliez pas de vous abonner|^\s*\.+\s*$",
re.I)
def _strip_whisper_artefacts(txt: str) -> str:
"""Drop whisper's silence/noise hallucinations so the NPC doesn't 'reply' to them."""
cleaned = _WHISPER_NOISE.sub("", txt or "").strip()
if not cleaned or _WHISPER_HALLUC.search(cleaned):
return ""
return cleaned
def _transcribe_whisper_sync(audio_wav: bytes) -> str:
conditioned = _normalize_wav_for_stt(audio_wav)
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
f.write(conditioned)
path = f.name
try:
out = subprocess.run(
[_WHISPER_BIN, "-m", _WHISPER_MODEL, "-f", path,
"-l", "fr", "-nt", "-np"],
capture_output=True, text=True, timeout=30,
)
return _strip_whisper_artefacts(out.stdout or "")
finally:
try:
os.unlink(path)
except OSError:
pass
async def _transcribe_whisper(audio_wav: bytes) -> str:
"""Transcribe a WAV via whisper.cpp turbo (high-pass conditioned), off-thread."""
return await asyncio.to_thread(_transcribe_whisper_sync, audio_wav)
def _ascii_header(s: str) -> str:
"""Make a string safe as an HTTP header value: ASCII-only AND no control
characters (newlines/CR/tab break the HTTP framing)."""
@@ -2087,7 +2133,7 @@ async def voice_reply(
try:
audio_bytes = await audio.read()
heard = await _transcribe_kyutai(http, audio_bytes)
heard = await _transcribe_whisper(audio_bytes)
logging.info("voice/reply[%s] HEARD: %r", number, heard)
VOICE_SESSIONS.append(session_id, "user", heard or "(inaudible)")
history = VOICE_SESSIONS.history(session_id)