feat(gateway): say TTS, local LLM, STT padding
- Replace Kyutai TTS with macOS say (16 kHz mono FR); NPC voice map _SAY_VOICE_MAP; helper _tts_say_sync - Local vllm-mlx :8520 as primary LLM (_chat_one), ailiance as fallback, spoken phrase as last resort; _chat_reply never raises (loop stays alive on 502) - Pad 800 ms silence before /transcribe (_pad_wav_silence) to prevent last-word truncation - Background _keep_llm_warm ping every 600 s (cold start) - Root log level INFO; X-Zacus-Heard/Said headers - test_chat_reply: empty_choices expects spoken fallback
This commit is contained in:
clement
@@ -42,8 +42,9 @@ def test_chat_reply_strips_whitespace():
|
||||
assert result == "Ici Zacus."
|
||||
|
||||
|
||||
def test_chat_reply_raises_on_empty_choices():
|
||||
"""ailiance returning {'choices': []} must raise RuntimeError, not KeyError/IndexError."""
|
||||
def test_chat_reply_falls_back_on_empty_choices():
|
||||
"""A malformed 200 ({'choices': []}) must not crash the voice loop: every
|
||||
backend fails the shape check, so _chat_reply returns the spoken fallback."""
|
||||
class FakeResponseEmpty:
|
||||
status_code = 200
|
||||
def json(self):
|
||||
@@ -53,8 +54,5 @@ def test_chat_reply_raises_on_empty_choices():
|
||||
async def post(self, url, *, json=None, timeout=None):
|
||||
return FakeResponseEmpty()
|
||||
|
||||
try:
|
||||
asyncio.run(main._chat_reply(FakeHTTPEmpty(), "persona", []))
|
||||
assert False, "expected RuntimeError"
|
||||
except RuntimeError as exc:
|
||||
assert "unexpected chat response shape" in str(exc)
|
||||
result = asyncio.run(main._chat_reply(FakeHTTPEmpty(), "persona", []))
|
||||
assert result == main._CHAT_FALLBACK
|
||||
|
||||
+126
-29
@@ -13,6 +13,8 @@ import os
|
||||
import re
|
||||
import secrets
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import types
|
||||
import wave
|
||||
@@ -118,6 +120,10 @@ class Settings(BaseSettings):
|
||||
ailiance_tts_model: str = "tts-1"
|
||||
ailiance_tts_voice: str = "nova"
|
||||
ailiance_chat_model: str = "gpt-4o-mini"
|
||||
# Local LLM (vllm-mlx on this Mac M1, OpenAI-compatible) — primary chat
|
||||
# backend for the NPC voice loop: native FR, fast, no flaky remote gateway.
|
||||
local_chat_url: str = "http://127.0.0.1:8520"
|
||||
local_chat_model: str = "granite4-tiny"
|
||||
kokoro_tts_url: str = "http://macm1:8520" # vllm-mlx native /v1/audio/speech (stock voices)
|
||||
kyutai_stt_url: str = "http://127.0.0.1:8300" # local Kyutai stt-1b-en_fr MLX server (/transcribe)
|
||||
kyutai_tts_url: str = "http://127.0.0.1:8302" # local Kyutai tts-1.6b-en_fr MLX server (/tts, native FR)
|
||||
@@ -207,25 +213,57 @@ _PHONE_STYLE = (
|
||||
)
|
||||
|
||||
|
||||
async def _chat_reply(http: httpx.AsyncClient, persona: str, history: list[dict]) -> str:
|
||||
"""Call ailiance chat completions with the NPC persona and return the reply text."""
|
||||
messages = [{"role": "system", "content": persona + _PHONE_STYLE}] + history
|
||||
# Spoken fallback when the LLM gateway is unreachable/slow: the conversational
|
||||
# loop must never leave the caller hanging on silence, so the NPC says a natural
|
||||
# "bad line, repeat please" instead of returning a 502.
|
||||
_CHAT_FALLBACK = "Excusez-moi, la ligne est très mauvaise, pouvez-vous répéter ?"
|
||||
|
||||
|
||||
async def _chat_one(http: httpx.AsyncClient, url: str, model: str,
|
||||
messages: list[dict], timeout: float) -> str:
|
||||
"""One OpenAI-compatible chat call. Raises on transport/HTTP/shape error."""
|
||||
resp = await http.post(
|
||||
f"{settings.ailiance_tts_url.rstrip('/')}/v1/chat/completions",
|
||||
json={"model": settings.ailiance_chat_model, "messages": messages, "max_tokens": 48},
|
||||
timeout=60.0,
|
||||
f"{url.rstrip('/')}/v1/chat/completions",
|
||||
json={"model": model, "messages": messages, "max_tokens": 48},
|
||||
timeout=timeout,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"ailiance chat HTTP {resp.status_code}: {resp.text[:160]}")
|
||||
data = resp.json()
|
||||
try:
|
||||
return data["choices"][0]["message"]["content"].strip()
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise RuntimeError("ailiance: unexpected chat response shape") from exc
|
||||
raise RuntimeError(f"chat HTTP {resp.status_code}: {resp.text[:160]}")
|
||||
return resp.json()["choices"][0]["message"]["content"].strip()
|
||||
|
||||
|
||||
async def _chat_reply(http: httpx.AsyncClient, persona: str, history: list[dict]) -> str:
|
||||
"""Produce the NPC reply text from the persona + conversation history.
|
||||
|
||||
Primary backend is the LOCAL vllm-mlx LLM on this Mac M1 (native FR, fast,
|
||||
no remote dependency); ailiance is a remote fallback if the local server is
|
||||
down. If both fail, return a spoken fallback so the voice loop never hangs.
|
||||
"""
|
||||
messages = [{"role": "system", "content": persona + _PHONE_STYLE}] + history
|
||||
backends = [
|
||||
# 60 s tolerates one cold model (re)load; a keep-warm task keeps it ~2 s.
|
||||
("local", settings.local_chat_url, settings.local_chat_model, 60.0),
|
||||
("ailiance", settings.ailiance_tts_url, settings.ailiance_chat_model, 12.0),
|
||||
]
|
||||
last_exc: Exception | None = None
|
||||
for name, url, model, timeout in backends:
|
||||
try:
|
||||
return await _chat_one(http, url, model, messages, timeout)
|
||||
except Exception as exc: # noqa: BLE001 — try the next backend on any failure
|
||||
last_exc = exc
|
||||
logging.warning("LLM backend %s failed (%s) — next", name, type(exc).__name__)
|
||||
logging.error("all LLM backends unreachable (%s) — spoken fallback", last_exc)
|
||||
return _CHAT_FALLBACK
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
# Surface our own INFO logs (HEARD/SAID transcriptions, prewarm) — uvicorn
|
||||
# only configures its own loggers, leaving the root logger at WARNING.
|
||||
root = logging.getLogger()
|
||||
if root.level > logging.INFO or not root.handlers:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
root.setLevel(logging.INFO)
|
||||
app.state.http = httpx.AsyncClient(timeout=settings.request_timeout)
|
||||
app.state.health_cache = HealthCache(app.state.http)
|
||||
app.state.audio_jobs = {} # job_id -> AudioJob (in-process async TTS queue)
|
||||
@@ -245,11 +283,28 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
except Exception as exc: # noqa: BLE001 — prewarm is best-effort
|
||||
logging.warning("TTS prewarm failed for %s: %s", number, str(exc)[:100])
|
||||
|
||||
async def _keep_llm_warm() -> None:
|
||||
"""Ping the local LLM periodically so it never auto-unloads/cools between
|
||||
calls — the first call after idle otherwise pays a ~40 s model reload.
|
||||
A tiny 1-token request loads (or keeps) the model resident."""
|
||||
while True:
|
||||
try:
|
||||
await _chat_one(
|
||||
app.state.http, settings.local_chat_url, settings.local_chat_model,
|
||||
[{"role": "user", "content": "ok"}], timeout=90.0,
|
||||
)
|
||||
logging.info("LLM keep-warm: %s resident", settings.local_chat_model)
|
||||
except Exception as exc: # noqa: BLE001 — best-effort
|
||||
logging.warning("LLM keep-warm failed: %s", type(exc).__name__)
|
||||
await asyncio.sleep(600) # 10 min < auto-unload-idle (1800 s)
|
||||
|
||||
prewarm_task = asyncio.create_task(_prewarm_greetings())
|
||||
warm_task = asyncio.create_task(_keep_llm_warm())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
prewarm_task.cancel()
|
||||
warm_task.cancel()
|
||||
await app.state.http.aclose()
|
||||
|
||||
|
||||
@@ -1765,47 +1820,87 @@ async def _tts_kyutai(http: httpx.AsyncClient, text: str, voice: str | None = No
|
||||
return resp.content
|
||||
|
||||
|
||||
# TTS is the latency bottleneck (Kyutai MLX ~0.3x realtime). Greetings are fixed
|
||||
# strings, so cache the finished 16 kHz WAV by (voice, text): first synth is slow,
|
||||
# every repeat is instant. Replies are dynamic and won't hit the cache.
|
||||
_TTS_CACHE: dict[str, tuple[bytes, float, bool]] = {}
|
||||
_TTS_CACHE_MAX = 64
|
||||
|
||||
# macOS `say` is the TTS engine: it emits a 16 kHz mono WAV directly, INSTANTLY
|
||||
# and reliably (no MLX, no slow synthesis, no warmup). The previous Kyutai MLX
|
||||
# TTS ran at ~0.3x realtime (~30-50 s/reply); `say` is ~real-time. French voice.
|
||||
SAY_VOICE_FR = "Jacques" # built-in fr_FR; `say -v '?'` lists others (Audrey, Eddy, Flo…)
|
||||
# Optional per-NPC voice variety: map directory voice tags → `say` voices.
|
||||
_SAY_VOICE_MAP = {"nova": "Audrey", "alloy": "Jacques"}
|
||||
|
||||
|
||||
def _tts_say_sync(text: str, voice: str) -> bytes:
|
||||
"""macOS `say` → 16 kHz mono 16-bit WAV bytes. Blocking (~real-time); call
|
||||
via asyncio.to_thread so the event loop isn't stalled."""
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
||||
path = tmp.name
|
||||
try:
|
||||
subprocess.run(
|
||||
["say", "-v", voice, "--file-format=WAVE",
|
||||
"--data-format=LEI16@16000", "-o", path, text],
|
||||
check=True, capture_output=True, timeout=30,
|
||||
)
|
||||
with open(path, "rb") as f:
|
||||
return f.read()
|
||||
finally:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
async def _voice_tts_16k(http: httpx.AsyncClient, text: str, voice: str) -> tuple[bytes, float, bool]:
|
||||
"""Synthesise text and resample to 16 kHz mono WAV for the PLIP.
|
||||
|
||||
Cached by (voice, text). Primary backend: local Kyutai TTS (native French);
|
||||
fallback: ailiance tts-1 if Kyutai is unreachable (anglophone, but not silent).
|
||||
"""
|
||||
"""Synthesise text to a 16 kHz mono WAV for the PLIP via macOS `say`, then
|
||||
run the dynamics chain (normalise/compress/limit) for a consistent level.
|
||||
Cached by (voice, text) — greetings are fixed so repeats are free."""
|
||||
key = f"{voice}|{text}"
|
||||
cached = _TTS_CACHE.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
try:
|
||||
raw = await _tts_kyutai(http, text, voice)
|
||||
except Exception as exc:
|
||||
logging.warning("Kyutai TTS unreachable (%s) — falling back to ailiance", str(exc)[:120])
|
||||
fake_body = types.SimpleNamespace(backend="ailiance", text=text, voice=voice)
|
||||
raw = await _synthesise_wav(http, fake_body) # type: ignore[arg-type]
|
||||
say_voice = _SAY_VOICE_MAP.get(voice, SAY_VOICE_FR)
|
||||
raw = await asyncio.to_thread(_tts_say_sync, text, say_voice)
|
||||
# say already emits 16 kHz mono → _wav_to_16k_mono skips resampling and just
|
||||
# applies the compressor + look-ahead limiter for a clean, even level.
|
||||
result = _wav_to_16k_mono(raw, max_seconds=7.5)
|
||||
|
||||
if len(_TTS_CACHE) >= _TTS_CACHE_MAX:
|
||||
_TTS_CACHE.clear() # crude bound — greetings are few, this rarely trips
|
||||
_TTS_CACHE.clear()
|
||||
_TTS_CACHE[key] = result
|
||||
return result
|
||||
|
||||
|
||||
def _pad_wav_silence(wav_bytes: bytes, ms: int = 800) -> bytes:
|
||||
"""Append `ms` of trailing silence to a WAV. Kyutai STT is streaming with a
|
||||
~0.5 s delay + semantic VAD; without an end-of-turn (trailing silence) the
|
||||
LAST words aren't flushed and the transcription is truncated. Padding acts as
|
||||
that end marker so the full utterance is transcribed."""
|
||||
try:
|
||||
with wave.open(io.BytesIO(wav_bytes), "rb") as w:
|
||||
params = w.getparams()
|
||||
raw = w.readframes(w.getnframes())
|
||||
n_silence = int(params.framerate * ms / 1000) * params.nchannels * params.sampwidth
|
||||
out = io.BytesIO()
|
||||
with wave.open(out, "wb") as w2:
|
||||
w2.setparams(params)
|
||||
w2.writeframes(raw + b"\x00" * n_silence)
|
||||
return out.getvalue()
|
||||
except Exception:
|
||||
return wav_bytes # malformed WAV → send as-is
|
||||
|
||||
|
||||
async def _transcribe_kyutai(http: httpx.AsyncClient, audio_wav: bytes) -> str:
|
||||
"""Transcribe a WAV via the local Kyutai stt-1b-en_fr MLX server (/transcribe).
|
||||
|
||||
The server resamples internally (Mimi @ 24 kHz), so any sample rate is fine.
|
||||
Pads trailing silence so Kyutai flushes the full utterance. The server
|
||||
resamples internally (Mimi @ 24 kHz), so any input rate is fine.
|
||||
Returns the recognised text (possibly empty on silence/inaudible input).
|
||||
"""
|
||||
resp = await http.post(
|
||||
f"{settings.kyutai_stt_url.rstrip('/')}/transcribe",
|
||||
content=audio_wav,
|
||||
content=_pad_wav_silence(audio_wav, 800),
|
||||
headers={"Content-Type": "audio/wav"},
|
||||
timeout=30.0,
|
||||
)
|
||||
@@ -1893,9 +1988,11 @@ async def voice_reply(
|
||||
try:
|
||||
audio_bytes = await audio.read()
|
||||
heard = await _transcribe_kyutai(http, audio_bytes)
|
||||
logging.info("voice/reply[%s] HEARD: %r", number, heard)
|
||||
VOICE_SESSIONS.append(session_id, "user", heard or "(inaudible)")
|
||||
history = VOICE_SESSIONS.history(session_id)
|
||||
said = await _chat_reply(http, persona, history)
|
||||
logging.info("voice/reply[%s] SAID: %r", number, said)
|
||||
VOICE_SESSIONS.append(session_id, "assistant", said)
|
||||
wav16, _, _ = await _voice_tts_16k(http, said, entry.get("voice", settings.ailiance_tts_voice))
|
||||
except HTTPException:
|
||||
|
||||
Reference in New Issue
Block a user