feat(voice): Whisper STT (robust to handset voice)
Switch the voice-loop STT from Kyutai to whisper.cpp large-v3-turbo (subprocess, high-pass conditioned): it decodes the weak/noisy SLIC handset captures Kyutai returned '' for. Filter whisper's silence/noise hallucinations (sous-titrage, radio-canada, tipeurs, '...') so the NPC never replies to a phantom line. Bump ESP32_ZACUS to 54022ed (VAD recalibration for the quiet voice + bell stop on pickup).
This commit was merged in pull request #167.
This commit is contained in:
+1
-1
Submodule ESP32_ZACUS updated: cfe429d885...54022ed6cc
@@ -2004,6 +2004,52 @@ async def _transcribe_kyutai(http: httpx.AsyncClient, audio_wav: bytes) -> str:
|
||||
return (resp.json().get("text") or "").strip()
|
||||
|
||||
|
||||
# whisper.cpp (large-v3-turbo, CoreML) — far more robust to the weak/noisy SLIC
|
||||
# handset captures than Kyutai's streaming STT (decodes voice Kyutai returns ''
|
||||
# for). Used as the primary STT for the voice loop.
|
||||
_WHISPER_BIN = "/Users/electron/whisperx-server/whisper.cpp/build/bin/whisper-cli"
|
||||
_WHISPER_MODEL = "/Users/electron/whisperx-server/models/ggml-large-v3-turbo-q5_0.bin"
|
||||
_WHISPER_NOISE = re.compile(r"\[(?:_?BLANK_AUDIO_?|BLANK_AUDIO|silence|musique|music)\]|\(\s*\)", re.I)
|
||||
# whisper hallucinates these training-data artefacts on silence/noise — treat as empty.
|
||||
_WHISPER_HALLUC = re.compile(
|
||||
r"sous-titr|société radio-canada|radio-canada|merci d'avoir regard|"
|
||||
r"amara\.org|abonnez-vous|tipeur|souscripteur|merci à tous|"
|
||||
r"n'oubliez pas de vous abonner|^\s*\.+\s*$",
|
||||
re.I)
|
||||
|
||||
|
||||
def _strip_whisper_artefacts(txt: str) -> str:
|
||||
"""Drop whisper's silence/noise hallucinations so the NPC doesn't 'reply' to them."""
|
||||
cleaned = _WHISPER_NOISE.sub("", txt or "").strip()
|
||||
if not cleaned or _WHISPER_HALLUC.search(cleaned):
|
||||
return ""
|
||||
return cleaned
|
||||
|
||||
|
||||
def _transcribe_whisper_sync(audio_wav: bytes) -> str:
|
||||
conditioned = _normalize_wav_for_stt(audio_wav)
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
||||
f.write(conditioned)
|
||||
path = f.name
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[_WHISPER_BIN, "-m", _WHISPER_MODEL, "-f", path,
|
||||
"-l", "fr", "-nt", "-np"],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
return _strip_whisper_artefacts(out.stdout or "")
|
||||
finally:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
async def _transcribe_whisper(audio_wav: bytes) -> str:
|
||||
"""Transcribe a WAV via whisper.cpp turbo (high-pass conditioned), off-thread."""
|
||||
return await asyncio.to_thread(_transcribe_whisper_sync, audio_wav)
|
||||
|
||||
|
||||
def _ascii_header(s: str) -> str:
|
||||
"""Make a string safe as an HTTP header value: ASCII-only AND no control
|
||||
characters (newlines/CR/tab break the HTTP framing)."""
|
||||
@@ -2087,7 +2133,7 @@ async def voice_reply(
|
||||
|
||||
try:
|
||||
audio_bytes = await audio.read()
|
||||
heard = await _transcribe_kyutai(http, audio_bytes)
|
||||
heard = await _transcribe_whisper(audio_bytes)
|
||||
logging.info("voice/reply[%s] HEARD: %r", number, heard)
|
||||
VOICE_SESSIONS.append(session_id, "user", heard or "(inaudible)")
|
||||
history = VOICE_SESSIONS.history(session_id)
|
||||
|
||||
Reference in New Issue
Block a user