feat(voice-bridge): /voice/ws auto-EOU safety

RMS-based silence detector on the receive loop closes the
utterance automatically when WS_VAD_SILENCE_TIMEOUT_MS of
contiguous silence follows at least WS_VAD_MIN_SPEECH_MS of
detected speech. Firmware's local VAD AFE still sends
{type:end} normally — this is a backstop for the cases where
the firmware loses the end frame or crashes mid-stream.

Two trigger paths:
  - frame-level: silence accumulates across incoming PCM frames
  - idle-timeout: no frame arrives at all for the silence
    window (firmware froze mid-stream)

Zero deps added — pure numpy RMS. Disable via WS_VAD_ENABLED=0;
tune thresholds via env. Smoke-tested with a client that never
sends {type:end}: server closes via auto-EOU, full pipeline runs.
This commit is contained in:
L'électron rare
2026-05-24 01:37:42 +02:00
parent 966ff731bd
commit 6d5edc7025
+93 -2
View File
@@ -184,6 +184,19 @@ WS_EXPECTED_SR = 16_000
WS_BYTES_PER_SAMPLE = 2 # PCM 16-bit mono
WS_MAX_DURATION_S = 30
WS_MAX_BYTES = WS_EXPECTED_SR * WS_BYTES_PER_SAMPLE * WS_MAX_DURATION_S # 960 KB
# Server-side end-of-utterance safety net (auto-EOU). Plain RMS-based
# silence detection: if we've seen at least WS_VAD_MIN_SPEECH_MS of
# speech (frames whose RMS exceeds WS_VAD_RMS_THRESHOLD) and then
# WS_VAD_SILENCE_TIMEOUT_MS of contiguous silence, we exit the
# receive loop as if the firmware had sent {"type":"end"}. Firmware's
# own VAD AFE remains primary — this is purely a backstop for the
# case where the firmware loses the end frame or crashes mid-stream.
# Set WS_VAD_ENABLED=0 to disable. Thresholds chosen empirically for
# the ESP32 INMP441 mic at typical desk distance — tune via env.
WS_VAD_ENABLED = os.getenv("WS_VAD_ENABLED", "1") not in {"0", "false", "no"}
WS_VAD_RMS_THRESHOLD = float(os.getenv("WS_VAD_RMS_THRESHOLD", "0.012"))
WS_VAD_MIN_SPEECH_MS = int(os.getenv("WS_VAD_MIN_SPEECH_MS", "400"))
WS_VAD_SILENCE_TIMEOUT_MS = int(os.getenv("WS_VAD_SILENCE_TIMEOUT_MS", "900"))
WS_FORWARD_INTENT = os.getenv("WS_FORWARD_INTENT", "1") not in {"0", "false", "no"}
# Server-side TTS over WS (P1 part12). Default: mirror WS_FORWARD_INTENT, since
# synthesizing the intent reply is the natural follow-up. Can be force-disabled
@@ -291,6 +304,23 @@ def _pcm16_seconds(pcm_bytes: int, sample_rate: int) -> float:
return pcm_bytes / float(sample_rate * 2)
def _pcm16_rms(pcm_bytes: bytes) -> float:
"""RMS amplitude of a PCM16 mono buffer, normalised to [0, 1].
Used by the server-side end-of-utterance safety net to detect
silence vs speech without pulling in a full VAD model. Empty
buffers return 0.0 (treated as silence).
"""
if not pcm_bytes:
return 0.0
samples = np.frombuffer(pcm_bytes, dtype=np.int16).astype(np.float32) / 32768.0
if samples.size == 0:
return 0.0
# Use mean-squared instead of sqrt(mean(x**2)) to avoid the sqrt on
# the hot path; the threshold is calibrated against this value.
return float(np.sqrt(np.mean(samples * samples)))
async def _usage_snapshot() -> dict[str, Any]:
"""Return a deep copy of the usage dict + computed ``uptime_s``."""
if _USAGE_LOCK is None:
@@ -1546,11 +1576,43 @@ async def voice_ws(ws: WebSocket) -> None:
_jlog("ws_hello_ok", request_id=request_id, session_id=session_id,
puzzle_id=puzzle_id)
# ── 2. Streaming loop: binary frames + text "end" ────────────────────
# ── 2. Streaming loop: binary frames + text "end" + server VAD ──────
buf = bytearray()
# Server-side EOU state: tracks how many ms of audio we've ingested
# since we last saw a frame above the speech threshold. Combined
# with `had_any_speech`, we trigger an automatic end-of-utterance
# when silence persists post-speech long enough. This is purely a
# safety net — the firmware's local VAD AFE still sends {"type":
# "end"} normally, and whichever signal arrives first wins.
audio_ms = 0 # total PCM duration accumulated, ms
last_speech_at_audio_ms = 0 # audio_ms when we last saw speech
had_any_speech = False
auto_eou_triggered = False
try:
while True:
msg = await ws.receive()
# If VAD is enabled and we've already locked in some speech,
# cap the per-frame await so a long-silent client unblocks
# the loop within WS_VAD_SILENCE_TIMEOUT_MS even when no
# further frames arrive at all (the firmware froze mid-stream).
if WS_VAD_ENABLED and had_any_speech:
remaining_silence_ms = max(
50,
WS_VAD_SILENCE_TIMEOUT_MS
- (audio_ms - last_speech_at_audio_ms),
)
try:
msg = await asyncio.wait_for(
ws.receive(), timeout=remaining_silence_ms / 1000.0
)
except asyncio.TimeoutError:
auto_eou_triggered = True
_jlog("ws_auto_eou_idle_timeout",
request_id=request_id, bytes=len(buf),
audio_ms=audio_ms,
since_speech_ms=audio_ms - last_speech_at_audio_ms)
break
else:
msg = await ws.receive()
# FastAPI / Starlette receive() returns a dict with either "bytes"
# or "text" populated; closing handshakes are surfaced as a
# "websocket.disconnect" type.
@@ -1565,6 +1627,35 @@ async def voice_ws(ws: WebSocket) -> None:
await ws.close(code=1009, reason="audio > 30 s cap")
return
buf.extend(data)
# Update VAD state on each binary frame. Frame duration
# depends on what the firmware ships (20 ms is the
# current default), but we derive it from byte count so
# any frame size works.
if WS_VAD_ENABLED:
frame_ms = int(
_pcm16_seconds(len(data), WS_EXPECTED_SR) * 1000
)
audio_ms += frame_ms
if _pcm16_rms(data) >= WS_VAD_RMS_THRESHOLD:
had_any_speech = True
last_speech_at_audio_ms = audio_ms
elif (
had_any_speech
and audio_ms - last_speech_at_audio_ms
>= WS_VAD_SILENCE_TIMEOUT_MS
and audio_ms >= WS_VAD_MIN_SPEECH_MS
):
# Enough trailing silence post-speech — close
# the utterance as if the firmware had sent end.
auto_eou_triggered = True
_jlog(
"ws_auto_eou_silence",
request_id=request_id, bytes=len(buf),
audio_ms=audio_ms,
since_speech_ms=audio_ms
- last_speech_at_audio_ms,
)
break
continue
if (text := msg.get("text")) is not None:
try: