feat(voice-bridge): STT post-correction aliases

Server-side fuzzy alias table applied to the final STT transcript
before forwarding to /voice/intent. Closes the metier-vocabulary
gap of generic STT models (U-SON, LA 440 Hz, Zacus, ...) without
needing per-request prompt biasing — which moshi-server's Rust
runtime does not expose anyway.

Loader is soft on pyyaml (degrades to no-op if missing) and on
the yaml file itself (logs + skips on parse error). Patterns are
case-insensitive with auto word-boundaries, with a (?...) escape
hatch for advanced regex.

Per-puzzle entries layer on top of globals — firmware passes
puzzle_id in the hello frame; legacy clients omit it and get
just the global polish. Loaded once at module import; restart
voice-bridge to reload.

Unit tested on 5 representative transcripts (eu son nez -> U-SON,
diapazon -> diapason, le rasus -> Zacus, le la 440 hertz -> LA 440 Hz).
This commit is contained in:
L'électron rare
2026-05-24 01:27:47 +02:00
parent 5c650049b2
commit 966ff731bd
3 changed files with 185 additions and 1 deletions
+62
View File
@@ -0,0 +1,62 @@
# STT post-correction fuzzy aliases for Kyutai (and any) STT output.
#
# The Kyutai 1B en_fr model does not know Zacus metier vocabulary
# (U-SON, the 440 Hz tuning fork context, custom NPC names, etc.) and
# the moshi-server Rust runtime does not expose any per-request
# TextPrompt mechanism we could use to bias the LM (cf.
# docs/voice-pipeline-roadmap.md). So we rewrite the transcript on the
# server side before forwarding to /voice/intent.
#
# Loaded by tools/macstudio/voice-bridge/main.py at boot via the
# `STT_ALIASES_PATH` env var (default: this file). Hot reload by
# restarting voice-bridge.
#
# Schema:
# global: # applied to every transcript
# - from: "regex pattern" # case-insensitive, word boundaries auto
# to: "replacement"
# per_puzzle: # keyed by puzzle_id (firmware passes it in hello)
# P1_SON:
# - from: "..."
# to: "..."
#
# Patterns are matched case-insensitively. Word boundaries are added
# around the pattern unless it starts with `(?` (escape hatch for
# bespoke regex). Replacements preserve case of the first matched
# character.
global:
# Professor name — the model often hears it as "the rasus" / "le rasus".
- from: "(?:le |la )?rasus"
to: "Zacus"
- from: "professeur ?(?:le |la )?rasus"
to: "Professeur Zacus"
# Common French STT polish — these are not Kyutai-specific but help
# everyone (e.g. whisper.cpp fallback would benefit too).
- from: "vue son"
to: "U-SON"
- from: "(?:eu|u) son nez"
to: "U-SON"
- from: "(?:eu|u) son(?:'s)?"
to: "U-SON"
- from: "husson(?: n'est)?"
to: "U-SON"
per_puzzle:
# Puzzle U-SON — frequency / tuning fork
SCENE_U_SON_PROTO:
- from: "(?:le |la )?la (?:quatre|4) ?cent ?quarante"
to: "LA 440"
- from: "(?:le |la )?la (?:quatre|4)40 ?(?:hertz|hz)"
to: "LA 440 Hz"
- from: "diapazon"
to: "diapason"
- from: "fréquence fondamentale"
to: "fréquence fondamentale"
# Reserve slots for the other scenes — populate as playtest reveals
# actual STT failure modes per puzzle.
SCENE_LA_DETECTOR: []
SCENE_MORSE: []
SCENE_COFFRE: []
+121 -1
View File
@@ -55,6 +55,7 @@ import asyncio
import hashlib
import io
import json
import re
import logging
import os
import shutil
@@ -97,6 +98,23 @@ WHISPER_URL = os.getenv("WHISPER_URL", "http://localhost:8300")
# reason to flip this to "whispercpp" is for A/B testing or if Kyutai
# is being upgraded.
STT_BACKEND = os.getenv("STT_BACKEND", "kyutai").lower()
# Post-correction fuzzy alias table for the STT output (cf.
# game/scenarios/stt_aliases.yaml). Loaded at boot, applied to the
# *final* transcript only (not the streaming partials, to avoid
# stuttering UI flicker on the firmware side).
STT_ALIASES_PATH = os.getenv(
"STT_ALIASES_PATH",
# First try sibling to main.py (Studio deploy puts the yaml next to
# the module), then fall back to the in-repo canonical location
# (dev/test from a checkout of the Zacus repo).
next(
(str(p) for p in [
Path(__file__).resolve().parent / "stt_aliases.yaml",
Path(__file__).resolve().parents[2] / "game" / "scenarios" / "stt_aliases.yaml",
] if p.exists()),
str(Path(__file__).resolve().parent / "stt_aliases.yaml"),
),
)
LITELLM_URL = os.getenv("LITELLM_URL", "http://localhost:4000")
LITELLM_DEFAULT_KEY = "sk-zacus-local-dev-do-not-share" # placeholder, log warns at boot
LITELLM_KEY = os.environ.get("LITELLM_MASTER_KEY", LITELLM_DEFAULT_KEY)
@@ -672,6 +690,94 @@ async def health_ready() -> JSONResponse:
return JSONResponse(content=body, status_code=code)
# ── STT post-correction fuzzy alias table ───────────────────────────────────
# Compiles once at module import. Format documented in
# game/scenarios/stt_aliases.yaml. Loader silently degrades to a no-op if
# the YAML can't be loaded — STT still works, just without metier polish.
_STT_GLOBAL_PATTERNS: list[tuple[re.Pattern[str], str]] = []
_STT_PUZZLE_PATTERNS: dict[str, list[tuple[re.Pattern[str], str]]] = {}
def _compile_alias_entries(entries: Any) -> list[tuple[re.Pattern[str], str]]:
"""Compile a list of {from, to} dicts into (Pattern, replacement) pairs.
Patterns get case-insensitive word-boundary wrapping unless they
already start with ``(?`` (escape hatch for advanced regex).
Invalid entries are logged and skipped — never raise.
"""
out: list[tuple[re.Pattern[str], str]] = []
if not isinstance(entries, list):
return out
for raw in entries:
if not isinstance(raw, dict):
continue
pat = raw.get("from")
repl = raw.get("to")
if not isinstance(pat, str) or not isinstance(repl, str):
continue
wrapped = pat if pat.startswith("(?") else rf"\b(?:{pat})\b"
try:
out.append((re.compile(wrapped, re.IGNORECASE), repl))
except re.error as exc:
_jlog("stt_alias_bad_regex", pattern=pat, err=str(exc))
return out
def _load_stt_aliases(path: str) -> None:
"""Populate the module-level alias tables. Idempotent."""
global _STT_GLOBAL_PATTERNS, _STT_PUZZLE_PATTERNS
try:
import yaml # local import — pyyaml is a soft dep
except ImportError:
_jlog("stt_alias_no_pyyaml")
return
p = Path(path)
if not p.exists():
_jlog("stt_alias_file_missing", path=str(p))
return
try:
data = yaml.safe_load(p.read_text())
except Exception as exc: # noqa: BLE001
_jlog("stt_alias_load_err", path=str(p), err=str(exc))
return
if not isinstance(data, dict):
return
_STT_GLOBAL_PATTERNS = _compile_alias_entries(data.get("global"))
per_puzzle = data.get("per_puzzle") or {}
_STT_PUZZLE_PATTERNS = {
str(k): _compile_alias_entries(v)
for k, v in per_puzzle.items()
if isinstance(v, list)
}
_jlog(
"stt_alias_loaded",
path=str(p),
global_count=len(_STT_GLOBAL_PATTERNS),
puzzle_keys=list(_STT_PUZZLE_PATTERNS.keys()),
)
def _apply_stt_aliases(text: str, puzzle_id: Optional[str] = None) -> str:
"""Apply global + puzzle-scoped substitutions to a final transcript.
No-op if alias tables are empty. Puzzle-scoped patterns apply on
top of globals so per-puzzle entries can refine generic ones.
"""
if not text:
return text
out = text
for pat, repl in _STT_GLOBAL_PATTERNS:
out = pat.sub(repl, out)
if puzzle_id:
for pat, repl in _STT_PUZZLE_PATTERNS.get(puzzle_id, ()):
out = pat.sub(repl, out)
return out
# Load at import so /voice/ws path has the tables ready when first WS hits.
_load_stt_aliases(STT_ALIASES_PATH)
# ── /tts (cache → F5 primary, Piper fallback, service_down last) ─────────────
async def _piper_fallback(text: str) -> Optional[bytes]:
"""POST text to Piper Tower :8001/synthesize. Returns WAV bytes or None.
@@ -1434,7 +1540,11 @@ async def voice_ws(ws: WebSocket) -> None:
await ws.close(code=1003, reason="unsupported hello")
return
session_id = hello.get("session_id")
_jlog("ws_hello_ok", request_id=request_id, session_id=session_id)
# Optional: firmware can pass the active puzzle_id so we apply
# puzzle-scoped STT alias overrides on top of the global table.
puzzle_id = hello.get("puzzle_id") if isinstance(hello.get("puzzle_id"), str) else None
_jlog("ws_hello_ok", request_id=request_id, session_id=session_id,
puzzle_id=puzzle_id)
# ── 2. Streaming loop: binary frames + text "end" ────────────────────
buf = bytearray()
@@ -1524,6 +1634,16 @@ async def voice_ws(ws: WebSocket) -> None:
# P1 part14: record STT call + audio duration (PCM16 16 kHz mono).
stt_audio_s = _pcm16_seconds(len(buf), WS_EXPECTED_SR)
await _usage_record_stt(stt_audio_s)
# Apply post-correction fuzzy alias table to the *final* transcript
# only (partials passed through verbatim above to avoid UI flicker).
# The result is what we forward to /voice/intent and what the
# firmware records — never a raw STT output with metier holes.
transcript_raw = transcript
transcript = _apply_stt_aliases(transcript_raw, puzzle_id=puzzle_id)
if transcript != transcript_raw:
_jlog("ws_stt_alias_applied", request_id=request_id,
before=transcript_raw[:160], after=transcript[:160],
puzzle_id=puzzle_id)
await ws.send_text(json.dumps(
{"type": "stt", "text": transcript, "final": True}
))
@@ -17,3 +17,5 @@ mlx>=0.31
msgpack>=1.0
websockets>=12
scipy>=1.10
# STT post-correction alias table loader (game/scenarios/stt_aliases.yaml).
pyyaml>=6