feat(hints): add LLM rewrite via LiteLLM hints-deep

This commit is contained in:
L'électron rare
2026-05-03 19:02:37 +02:00
parent e8a5964413
commit 1ed230ebd0
4 changed files with 633 additions and 74 deletions
+1 -1
View File
@@ -71,7 +71,7 @@ playtest:
--snapshot game/scenarios/playtests/snapshots/zacus_v3_60min_tech.snapshot.json
hints-serve:
uv run --with fastapi --with uvicorn --with pyyaml --with pydantic \
uv run --with fastapi --with uvicorn --with pyyaml --with pydantic --with httpx \
uvicorn tools.hints.server:app --reload --port 8300
hints-test:
+41
View File
@@ -0,0 +1,41 @@
# Hints safety filter — banned words/numbers per puzzle.
#
# When the LLM rewrites a hint (Phase P2 of hints engine), its output is
# scanned for any token in the per-puzzle banned list (case-insensitive,
# accent-insensitive). If a hit is detected, the rewrite is rejected and
# the engine falls back to the verbatim base phrase.
#
# The intent is to prevent the LLM from accidentally leaking the literal
# solution while it is rewording a level-1 or level-2 hint. Bans here are
# intentionally narrow — only the *answer* tokens, not the puzzle vocabulary.
# Default (no entry for a puzzle): no filtering.
#
# Spec: docs/superpowers/specs/2026-05-03-hints-engine-design.md §4 ("Safety rail")
banned_per_puzzle:
# Sound puzzle: 5-button colour sequence Rouge/Bleu/Jaune/Rouge/Vert.
# The full ordered solution lives at level_3; we forbid the rewriter from
# echoing the full sequence at level_1 or level_2.
P1_SON:
- "rouge bleu jaune"
- "bleu jaune rouge vert"
# QR cache hunt: the level_3 hint enumerates the 6-cache scan order.
# Any of these landmark words leaking would skip the deduction.
P3_QR:
- "etagere gauche"
- "tiroir rouge"
- "boite bleue"
- "box-3"
# Radio frequency: the answer is literally 1337 Hz ("LEET").
P4_RADIO:
- "1337"
- "leet"
# Morse code: the message is "ZACUS". The level_3 hint also contains the
# raw dot/dash transcription. Rewriter must not pre-spell either.
P5_MORSE:
- "zacus"
- "z=--.."
- "tiret-tiret-point"
+207 -20
View File
@@ -1,15 +1,18 @@
"""Tests for tools.hints.server (P1 static phrase lookup).
"""Tests for tools.hints.server (P1 static lookup + P2 LLM rewrite).
Run via:
make hints-test
or:
uv run --with fastapi --with uvicorn --with pyyaml --with pydantic \
--with pytest --with httpx pytest tests/hints/ -v
--with pytest --with httpx --with pytest-asyncio \
pytest tests/hints/ -v
"""
from __future__ import annotations
import asyncio
import sys
from pathlib import Path
from typing import Any, Dict, List, Tuple
import pytest
from fastapi.testclient import TestClient
@@ -17,10 +20,13 @@ from fastapi.testclient import TestClient
REPO_ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(REPO_ROOT))
from tools.hints.server import ( # noqa: E402 (import after sys.path tweak)
from tools.hints import server as server_module # noqa: E402
from tools.hints.server import ( # noqa: E402
DEFAULT_PHRASES_PATH,
DEFAULT_SAFETY_PATH,
create_app,
load_phrase_bank,
safety_check,
)
@@ -31,7 +37,7 @@ from tools.hints.server import ( # noqa: E402 (import after sys.path tweak)
@pytest.fixture(scope="module")
def app():
return create_app(DEFAULT_PHRASES_PATH)
return create_app(DEFAULT_PHRASES_PATH, DEFAULT_SAFETY_PATH)
@pytest.fixture(scope="module")
@@ -43,8 +49,7 @@ def client(app):
def real_puzzle_id():
"""First puzzle in the bank that has a non-empty level_1.
Using a real puzzle from the actual YAML keeps the test honest — if
`npc_phrases.yaml` drifts, the suite catches it.
Using a real puzzle from the actual YAML keeps the test honest.
"""
bank = load_phrase_bank(DEFAULT_PHRASES_PATH)
for pid, levels in bank.items():
@@ -53,8 +58,31 @@ def real_puzzle_id():
pytest.fail("no puzzle with level_1 found in npc_phrases.yaml")
def _make_fake_call(
content: str = "Mon cher, écoutez le rythme et comptez les battements.",
*,
raise_exc: Exception | None = None,
delay_s: float = 0.0,
tokens_used: int = 42,
):
"""Build a coroutine that mimics `_call_litellm` for monkeypatching."""
async def fake(**kwargs: Any) -> Tuple[str, Dict[str, Any]]:
if delay_s:
await asyncio.sleep(delay_s)
if raise_exc is not None:
raise raise_exc
raw = {
"choices": [{"message": {"content": content}}],
"usage": {"total_tokens": tokens_used},
}
return content, raw
return fake
# ---------------------------------------------------------------------------
# Health + coverage
# Health + coverage (P1 + P2 health surface)
# ---------------------------------------------------------------------------
@@ -66,6 +94,10 @@ def test_healthz_returns_ok_with_counts(client):
assert body["phrases_loaded"] > 0
assert body["puzzles_loaded"] > 0
assert body["phrases_path"].endswith("npc_phrases.yaml")
# P2 fields
assert "safety_puzzles_loaded" in body
assert "litellm_url" in body
assert body["llm_model"]
def test_coverage_summary_shape_is_complete(client):
@@ -73,17 +105,14 @@ def test_coverage_summary_shape_is_complete(client):
assert resp.status_code == 200
body = resp.json()
# Required top-level fields
for key in ("total_puzzles", "puzzles_by_level_count",
"percent_by_level_count", "per_puzzle"):
assert key in body, f"coverage missing field {key!r}"
# Bucket counts must sum to total
bucket_sum = sum(body["puzzles_by_level_count"].values())
assert bucket_sum == body["total_puzzles"]
assert body["total_puzzles"] >= 1
# Each per_puzzle row has the expected sub-fields
for row in body["per_puzzle"]:
assert "puzzle_id" in row
assert "levels_present" in row
@@ -93,13 +122,13 @@ def test_coverage_summary_shape_is_complete(client):
# ---------------------------------------------------------------------------
# /hints/ask happy path + errors
# /hints/ask happy path + errors (verbatim path with ?rewrite=false)
# ---------------------------------------------------------------------------
def test_hints_ask_happy_path_returns_real_phrase(client, real_puzzle_id):
resp = client.post(
"/hints/ask",
"/hints/ask?rewrite=false",
json={"puzzle_id": real_puzzle_id, "level": 1, "session_id": "test-session"},
)
assert resp.status_code == 200, resp.text
@@ -107,18 +136,19 @@ def test_hints_ask_happy_path_returns_real_phrase(client, real_puzzle_id):
assert body["puzzle_id"] == real_puzzle_id
assert body["level"] == 1
assert body["source"] == "static"
assert body["model_used"] == "none"
assert body["hint_rewritten"] is None
assert isinstance(body["hint"], str)
assert len(body["hint"]) > 0
assert body["hint"] == body["hint_static"]
# The phrase must be one of the real phrases in the bank — guards
# against accidental string mangling in the loader.
bank = load_phrase_bank(DEFAULT_PHRASES_PATH)
assert body["hint"] in bank[real_puzzle_id]["level_1"]
def test_hints_ask_unknown_puzzle_returns_404(client):
resp = client.post(
"/hints/ask",
"/hints/ask?rewrite=false",
json={"puzzle_id": "P999_DOES_NOT_EXIST", "level": 1},
)
assert resp.status_code == 404
@@ -126,16 +156,15 @@ def test_hints_ask_unknown_puzzle_returns_404(client):
def test_hints_ask_invalid_level_returns_422(client, real_puzzle_id):
# Pydantic validation rejects out-of-range level → 422 (Unprocessable)
resp = client.post(
"/hints/ask",
"/hints/ask?rewrite=false",
json={"puzzle_id": real_puzzle_id, "level": 7},
)
assert resp.status_code == 422
def test_hints_ask_preserves_french_diacritics(client, real_puzzle_id):
"""UTF-8 round-trip must keep accented characters intact."""
"""UTF-8 round-trip must keep accented characters intact (verbatim path)."""
bank = load_phrase_bank(DEFAULT_PHRASES_PATH)
has_accent = any(
any(ch in phrase for ch in "éèêàùôîçÉÈÊÀÙÔÎÇ")
@@ -145,14 +174,172 @@ def test_hints_ask_preserves_french_diacritics(client, real_puzzle_id):
if not has_accent:
pytest.skip(f"{real_puzzle_id} has no accented phrases — skipping diacritic check")
# Try several requests — at least one should land on an accented phrase
found_accent = False
for _ in range(20):
resp = client.post(
"/hints/ask", json={"puzzle_id": real_puzzle_id, "level": 1}
"/hints/ask?rewrite=false", json={"puzzle_id": real_puzzle_id, "level": 1}
)
assert resp.status_code == 200
if any(ch in resp.json()["hint"] for ch in "éèêàùôîçÉÈÊÀÙÔÎÇ"):
found_accent = True
break
assert found_accent, "diacritics never round-tripped — encoding bug?"
# ---------------------------------------------------------------------------
# Safety filter unit
# ---------------------------------------------------------------------------
def test_safety_check_is_accent_and_case_insensitive():
banned = ["Étagère gauche", "1337"]
assert safety_check("rien à signaler", banned) is None
assert safety_check("vise l'ETAGERE GAUCHE rapidement", banned) == "Étagère gauche"
assert safety_check("la fréquence est 1337 hertz", banned) == "1337"
assert safety_check("anything", []) is None
# ---------------------------------------------------------------------------
# /hints/ask P2 — LLM rewrite paths (mocked LiteLLM)
# ---------------------------------------------------------------------------
def test_ask_with_rewrite_uses_llm(client, real_puzzle_id, monkeypatch):
"""Rewrite path: mocked LLM returns clean content → source=llm_rewritten."""
fake = _make_fake_call(
content="Ah, mon cher disciple ! Écoutez ce rythme avec ferveur.",
tokens_used=37,
)
monkeypatch.setattr(server_module, "_call_litellm", fake)
resp = client.post(
"/hints/ask",
json={"puzzle_id": real_puzzle_id, "level": 1, "session_id": "rw-test"},
)
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["source"] == "llm_rewritten"
assert body["model_used"] == client.app.state.llm_model
assert body["hint_rewritten"] == "Ah, mon cher disciple ! Écoutez ce rythme avec ferveur."
assert body["hint"] == body["hint_rewritten"]
assert isinstance(body["hint_static"], str) and len(body["hint_static"]) > 0
assert body["hint_static"] != body["hint_rewritten"]
def test_ask_rewrite_fallback_on_timeout(client, real_puzzle_id, monkeypatch):
"""asyncio.TimeoutError from `_call_litellm` → verbatim fallback."""
fake = _make_fake_call(raise_exc=asyncio.TimeoutError())
monkeypatch.setattr(server_module, "_call_litellm", fake)
resp = client.post(
"/hints/ask",
json={"puzzle_id": real_puzzle_id, "level": 1},
)
assert resp.status_code == 200
body = resp.json()
assert body["source"] == "llm_fallback_static"
assert body["model_used"] == "none"
assert body["hint_rewritten"] is None
assert body["hint"] == body["hint_static"]
def test_ask_rewrite_fallback_on_safety_trigger(client, monkeypatch):
"""LLM emits a banned word → safety filter trips → verbatim fallback."""
# P4_RADIO bans "1337" — make the LLM "leak" it.
fake = _make_fake_call(
content="Tournez le bouton jusqu'à ce que l'écran affiche 1337 hertz précisément.",
)
monkeypatch.setattr(server_module, "_call_litellm", fake)
resp = client.post(
"/hints/ask",
json={"puzzle_id": "P4_RADIO", "level": 1},
)
assert resp.status_code == 200
body = resp.json()
assert body["source"] == "llm_fallback_static"
assert body["model_used"] == "none"
assert body["hint_rewritten"] is None
assert body["hint"] == body["hint_static"]
# Sanity: the static phrase itself isn't the leaky one (it stays in bank as-is)
assert isinstance(body["hint_static"], str) and len(body["hint_static"]) > 0
def test_ask_query_rewrite_false_bypasses_llm(client, real_puzzle_id, monkeypatch):
"""`?rewrite=false` must short-circuit and never call _call_litellm."""
called = {"n": 0}
async def explode(**kwargs: Any):
called["n"] += 1
raise AssertionError("LLM was called despite rewrite=false")
monkeypatch.setattr(server_module, "_call_litellm", explode)
resp = client.post(
"/hints/ask?rewrite=false",
json={"puzzle_id": real_puzzle_id, "level": 1},
)
assert resp.status_code == 200
body = resp.json()
assert called["n"] == 0
assert body["source"] == "static"
assert body["model_used"] == "none"
assert body["hint_rewritten"] is None
def test_ask_rewrite_fallback_on_backend_error(client, real_puzzle_id, monkeypatch):
"""Generic backend RuntimeError (HTTP 500, malformed JSON…) → fallback."""
fake = _make_fake_call(raise_exc=RuntimeError("litellm http 500: oops"))
monkeypatch.setattr(server_module, "_call_litellm", fake)
resp = client.post(
"/hints/ask",
json={"puzzle_id": real_puzzle_id, "level": 1},
)
assert resp.status_code == 200
body = resp.json()
assert body["source"] == "llm_fallback_static"
assert body["hint"] == body["hint_static"]
# ---------------------------------------------------------------------------
# /hints/rewrite debug endpoint
# ---------------------------------------------------------------------------
def test_rewrite_endpoint_works(client, real_puzzle_id, monkeypatch):
fake = _make_fake_call(
content="Mon cher, ne sous-estimez jamais la mémoire d'une mélodie ancienne.",
tokens_used=29,
)
monkeypatch.setattr(server_module, "_call_litellm", fake)
resp = client.post(
"/hints/rewrite",
json={
"puzzle_id": real_puzzle_id,
"level": 1,
"max_tokens": 64,
"temperature": 0.5,
},
)
assert resp.status_code == 200, resp.text
body = resp.json()
assert body["puzzle_id"] == real_puzzle_id
assert body["level"] == 1
assert body["model_used"] == client.app.state.llm_model
assert body["hint_rewritten"].startswith("Mon cher")
assert body["tokens_used"] == 29
assert isinstance(body["hint_static"], str) and len(body["hint_static"]) > 0
def test_rewrite_endpoint_502_on_safety_block(client, monkeypatch):
fake = _make_fake_call(content="La fréquence est 1337 hertz, sans hésitation.")
monkeypatch.setattr(server_module, "_call_litellm", fake)
resp = client.post(
"/hints/rewrite",
json={"puzzle_id": "P4_RADIO", "level": 1},
)
assert resp.status_code == 502
assert "safety" in resp.json()["detail"].lower()
+384 -53
View File
@@ -1,21 +1,37 @@
"""Hints engine — Phase P1 FastAPI server.
"""Hints engine — Phase P2 FastAPI server.
Static phrase lookup (P1) + LLM rewrite layer via LiteLLM `hints-deep` (P2).
The rewriter restyles the static base phrase in the Professor Zacus voice
without inventing new content. Safety filter blocks rewrites that leak
solution tokens. On timeout / error / safety trip, the engine falls back
to the verbatim static phrase so the game never blocks.
Stateless static phrase lookup from `game/scenarios/npc_phrases.yaml`.
Endpoints:
- GET /healthz — liveness probe + phrases-loaded count
- POST /hints/ask — return base phrase for {puzzle_id, level}
- GET /hints/coverage — per-puzzle level coverage audit
- GET /healthz — liveness probe + phrases-loaded count
- POST /hints/ask — return base phrase, optionally LLM-rewritten
(?rewrite=false to bypass LLM)
- POST /hints/rewrite — debug: rewrite-only path, no static fallback
- GET /hints/coverage — per-puzzle level coverage audit
Run:
uv run --with fastapi --with uvicorn --with pyyaml --with pydantic \
--with httpx \
uvicorn tools.hints.server:app --reload --port 8300
Spec: docs/superpowers/specs/2026-05-03-hints-engine-design.md
P1 scope is intentionally narrow: no LLM rewrite, no anti-cheat,
no adaptive level selection, no auto-trigger. Those land in P2-P6.
Env:
LITELLM_URL (default http://192.168.0.120:4000)
LITELLM_MASTER_KEY (default sk-zacus-local-dev-do-not-share)
HINTS_LLM_MODEL (default hints-deep)
HINTS_LLM_TIMEOUT_S (default 8.0)
HINTS_SAFETY_PATH (default game/scenarios/hints_safety.yaml)
ZACUS_NPC_PHRASES_PATH (default game/scenarios/npc_phrases.yaml)
Spec: docs/superpowers/specs/2026-05-03-hints-engine-design.md §4
P3 (anti-cheat) and P4 (adaptive level) are intentionally NOT in scope here.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
@@ -23,11 +39,13 @@ import random
import re
import sys
import time
import unicodedata
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Tuple
import httpx
import yaml
from fastapi import FastAPI, HTTPException, Request
from fastapi import FastAPI, HTTPException, Query, Request
from pydantic import BaseModel, Field, conint
@@ -37,15 +55,26 @@ from pydantic import BaseModel, Field, conint
REPO_ROOT = Path(__file__).resolve().parents[2]
DEFAULT_PHRASES_PATH = REPO_ROOT / "game" / "scenarios" / "npc_phrases.yaml"
DEFAULT_SAFETY_PATH = REPO_ROOT / "game" / "scenarios" / "hints_safety.yaml"
PHRASES_PATH_ENV = "ZACUS_NPC_PHRASES_PATH"
SAFETY_PATH_ENV = "HINTS_SAFETY_PATH"
DEFAULT_LITELLM_URL = "http://192.168.0.120:4000"
DEFAULT_LITELLM_KEY = "sk-zacus-local-dev-do-not-share"
DEFAULT_LLM_MODEL = "hints-deep"
DEFAULT_LLM_TIMEOUT_S = 8.0
DEFAULT_LLM_MAX_TOKENS = 80
DEFAULT_LLM_TEMPERATURE = 0.4
ZACUS_SYSTEM_PROMPT = (
"Tu es le Professeur Zacus, savant excentrique. Réécris l'indice "
"ci-dessous en restant fidèle à son contenu mais avec ton style "
"théâtral et un ton dramatique adapté à un escape room. 2 phrases "
"max. Ne révèle pas la solution, garde les nombres, codes et noms "
"propres exacts. Pas d'emoji."
)
# Puzzle IDs that look like V3 numbered puzzles (P1_FOO, P12_BAR, ...).
# Authoring drift: `P1_SON`..`P7_COFFRE` are nested under `endings:` in the
# current YAML (indentation bug — see audit). We surface them anyway because
# their internal `key:` values advertise `hints.<id>.level_N`. Fix later in
# `npc_phrases.yaml`; do NOT fix here (P1 instructions forbid YAML edits).
PUZZLE_ID_PATTERN = re.compile(r"^P\d+_[A-Z0-9_]+$")
ALL_LEVELS = (1, 2, 3)
LOG = logging.getLogger("hints.server")
@@ -80,7 +109,6 @@ def _normalize_level_block(block: Any) -> List[str]:
if isinstance(block, str):
return [block.strip()] if block.strip() else []
if isinstance(block, dict):
# Tolerate `{text: "..."}` at the level position
text = _extract_text(block).strip()
return [text] if text else []
return []
@@ -95,10 +123,7 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
2. `endings.<puzzle_id>.level_N` where puzzle_id matches PUZZLE_ID_PATTERN
(the V3 P1-P7 hints currently mis-nested under `endings:`)
Raises:
FileNotFoundError if `path` doesn't exist.
ValueError if the YAML lacks a top-level `hints` mapping or no usable
puzzle entries are found.
Raises FileNotFoundError or ValueError on schema problems.
"""
if not path.is_file():
raise FileNotFoundError(f"npc_phrases.yaml not found at {path}")
@@ -115,7 +140,6 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
bank: Dict[str, Dict[str, List[str]]] = {}
# 1. Canonical hints.* puzzles
for puzzle_id, levels in hints_section.items():
if not isinstance(levels, dict):
continue
@@ -124,7 +148,6 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
for n in ALL_LEVELS
}
# 2. Mis-nested V3 puzzles under endings.*
endings_section = data.get("endings")
if isinstance(endings_section, dict):
for puzzle_id, levels in endings_section.items():
@@ -133,7 +156,7 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
if not isinstance(levels, dict):
continue
if puzzle_id in bank:
continue # canonical hints.* wins
continue
bank[puzzle_id] = {
f"level_{n}": _normalize_level_block(levels.get(f"level_{n}"))
for n in ALL_LEVELS
@@ -148,6 +171,32 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
return bank
def load_safety_bank(path: Path) -> Dict[str, List[str]]:
"""Load `hints_safety.yaml` and return `{puzzle_id: [banned_word, ...]}`.
Empty mapping if the file is absent (P2 default, no filtering anywhere).
Banned words are stored as-is; comparison is normalized at filter time.
"""
if not path.is_file():
LOG.info("hints_safety.yaml not found at %s — running with no safety filter", path)
return {}
with path.open("r", encoding="utf-8") as f:
data = yaml.safe_load(f) or {}
if not isinstance(data, dict):
return {}
raw = data.get("banned_per_puzzle") or {}
if not isinstance(raw, dict):
return {}
out: Dict[str, List[str]] = {}
for puzzle_id, words in raw.items():
if not isinstance(words, list):
continue
cleaned = [str(w).strip() for w in words if isinstance(w, (str, int, float)) and str(w).strip()]
if cleaned:
out[str(puzzle_id)] = cleaned
return out
def compute_coverage(bank: Dict[str, Dict[str, List[str]]]) -> Dict[str, Any]:
"""Build the audit summary returned by GET /hints/coverage."""
per_puzzle: List[Dict[str, Any]] = []
@@ -182,6 +231,89 @@ def count_phrases(bank: Dict[str, Dict[str, List[str]]]) -> int:
return sum(len(level) for puzzle in bank.values() for level in puzzle.values())
# ---------------------------------------------------------------------------
# Safety filter
# ---------------------------------------------------------------------------
def _strip_accents(s: str) -> str:
"""Lower + strip diacritics for case/accent-insensitive matching."""
nfkd = unicodedata.normalize("NFKD", s)
return "".join(ch for ch in nfkd if not unicodedata.combining(ch)).lower()
def safety_check(text: str, banned_words: List[str]) -> Optional[str]:
"""Return the first matched banned word if `text` contains one, else None.
Matching is case- and accent-insensitive on a normalized substring.
Empty `banned_words` means "no filter" → always None.
"""
if not banned_words:
return None
haystack = _strip_accents(text)
for word in banned_words:
needle = _strip_accents(word)
if needle and needle in haystack:
return word
return None
# ---------------------------------------------------------------------------
# LiteLLM call (mockable in tests via monkeypatch)
# ---------------------------------------------------------------------------
def _build_messages(hint_static: str, puzzle_id: str, level: int) -> List[Dict[str, str]]:
user_prompt = (
f"Niveau {level}, énigme {puzzle_id}. "
f"Phrase originale à reformuler : « {hint_static} »"
)
return [
{"role": "system", "content": ZACUS_SYSTEM_PROMPT},
{"role": "user", "content": user_prompt},
]
async def _call_litellm(
*,
base_url: str,
api_key: str,
model: str,
messages: List[Dict[str, str]],
timeout_s: float,
max_tokens: int,
temperature: float,
) -> Tuple[str, Dict[str, Any]]:
"""POST to LiteLLM `/v1/chat/completions`, return (content, raw_json).
Raises `asyncio.TimeoutError`, `httpx.HTTPError`, or `RuntimeError` on
a non-200 / malformed response. Callers MUST catch and fall back.
Tests monkeypatch this whole function to avoid network I/O.
"""
payload = {
"model": model,
"messages": messages,
"max_tokens": max_tokens,
"temperature": temperature,
}
async with httpx.AsyncClient(timeout=timeout_s) as client:
resp = await client.post(
f"{base_url}/v1/chat/completions",
headers={"Authorization": f"Bearer {api_key}"},
json=payload,
)
if resp.status_code != 200:
raise RuntimeError(f"litellm http {resp.status_code}: {resp.text[:200]}")
data = resp.json()
try:
content = data["choices"][0]["message"]["content"].strip()
except (KeyError, IndexError, TypeError) as exc:
raise RuntimeError(f"litellm malformed response: {exc}") from exc
if not content:
raise RuntimeError("litellm returned empty content")
return content, data
# ---------------------------------------------------------------------------
# Pydantic request/response models
# ---------------------------------------------------------------------------
@@ -195,9 +327,31 @@ class HintAskRequest(BaseModel):
class HintAskResponse(BaseModel):
hint: str
hint_static: str
hint_rewritten: Optional[str] = None
level: int
puzzle_id: str
source: str = "static"
source: str # "static" | "llm_rewritten" | "llm_fallback_static"
model_used: str # "hints-deep" | "none"
latency_ms: float
class HintRewriteRequest(BaseModel):
puzzle_id: str = Field(..., min_length=1)
level: conint(ge=1, le=3) # type: ignore[valid-type]
session_id: Optional[str] = None
max_tokens: Optional[conint(ge=8, le=512)] = None # type: ignore[valid-type]
temperature: Optional[float] = Field(None, ge=0.0, le=2.0)
class HintRewriteResponse(BaseModel):
hint_static: str
hint_rewritten: str
level: int
puzzle_id: str
model_used: str
latency_ms: float
tokens_used: Optional[int] = None
# ---------------------------------------------------------------------------
@@ -210,30 +364,120 @@ def _resolve_phrases_path() -> Path:
return Path(override).resolve() if override else DEFAULT_PHRASES_PATH
def _resolve_safety_path() -> Path:
override = os.environ.get(SAFETY_PATH_ENV)
return Path(override).resolve() if override else DEFAULT_SAFETY_PATH
def _emit_log(record: Dict[str, Any]) -> None:
"""Structured JSON log line on stdout (one dict per request)."""
sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n")
sys.stdout.flush()
def create_app(phrases_path: Optional[Path] = None) -> FastAPI:
"""Build the FastAPI app. Pass `phrases_path` for tests, default for prod."""
path = phrases_path or _resolve_phrases_path()
bank = load_phrase_bank(path) # raises if cracked → uvicorn refuses to start
def create_app(
phrases_path: Optional[Path] = None,
safety_path: Optional[Path] = None,
) -> FastAPI:
"""Build the FastAPI app. Pass paths for tests, defaults for prod."""
p_path = phrases_path or _resolve_phrases_path()
s_path = safety_path or _resolve_safety_path()
bank = load_phrase_bank(p_path)
safety = load_safety_bank(s_path)
coverage_cache = compute_coverage(bank)
phrase_total = count_phrases(bank)
app = FastAPI(
title="Zacus Hints Engine",
version="0.1.0-P1",
description="Static NPC hint phrase lookup. Phase P1 of the hints engine spec.",
version="0.2.0-P2",
description=(
"Static NPC hint phrase lookup + LLM rewrite via LiteLLM hints-deep. "
"Phase P2 of the hints engine spec."
),
)
# Stash for handlers + tests
app.state.phrase_bank = bank
app.state.safety_bank = safety
app.state.coverage = coverage_cache
app.state.phrase_total = phrase_total
app.state.phrases_path = str(path)
app.state.phrases_path = str(p_path)
app.state.safety_path = str(s_path)
app.state.litellm_url = os.environ.get("LITELLM_URL", DEFAULT_LITELLM_URL)
app.state.litellm_key = os.environ.get("LITELLM_MASTER_KEY", DEFAULT_LITELLM_KEY)
app.state.llm_model = os.environ.get("HINTS_LLM_MODEL", DEFAULT_LLM_MODEL)
app.state.llm_timeout_s = float(
os.environ.get("HINTS_LLM_TIMEOUT_S", str(DEFAULT_LLM_TIMEOUT_S))
)
def _resolve_static(puzzle_id: str, level: int) -> str:
puzzle = app.state.phrase_bank.get(puzzle_id)
if puzzle is None:
raise HTTPException(
status_code=404,
detail=f"unknown puzzle_id: {puzzle_id!r}",
)
level_key = f"level_{level}"
phrases = puzzle.get(level_key) or []
if not phrases:
raise HTTPException(
status_code=404,
detail=(
f"no phrases for {puzzle_id} at {level_key} "
f"(available: {[k for k, v in puzzle.items() if v]})"
),
)
return random.choice(phrases)
async def _try_rewrite(
hint_static: str,
puzzle_id: str,
level: int,
*,
max_tokens: int = DEFAULT_LLM_MAX_TOKENS,
temperature: float = DEFAULT_LLM_TEMPERATURE,
) -> Tuple[Optional[str], str, Optional[str], Optional[int]]:
"""Best-effort LLM rewrite.
Returns (rewritten_text_or_none, source_tag, safety_trigger_word_or_none,
tokens_used_or_none). `source_tag` is "llm_rewritten" on success and
"llm_fallback_static" on any failure (timeout / error / safety trip).
"""
messages = _build_messages(hint_static, puzzle_id, level)
try:
content, raw = await asyncio.wait_for(
_call_litellm(
base_url=app.state.litellm_url,
api_key=app.state.litellm_key,
model=app.state.llm_model,
messages=messages,
timeout_s=app.state.llm_timeout_s,
max_tokens=max_tokens,
temperature=temperature,
),
timeout=app.state.llm_timeout_s,
)
except asyncio.TimeoutError:
LOG.warning("hints rewrite timeout for %s/level_%d", puzzle_id, level)
return None, "llm_fallback_static", None, None
except Exception as exc: # noqa: BLE001 fallback covers all backend issues
LOG.warning("hints rewrite error for %s/level_%d: %s", puzzle_id, level, exc)
return None, "llm_fallback_static", None, None
tokens_used = None
usage = (raw or {}).get("usage") if isinstance(raw, dict) else None
if isinstance(usage, dict):
tokens_used = usage.get("total_tokens")
banned = app.state.safety_bank.get(puzzle_id, [])
trigger = safety_check(content, banned)
if trigger:
LOG.warning(
"hints safety filter triggered for %s/level_%d on word=%r",
puzzle_id, level, trigger,
)
return None, "llm_fallback_static", trigger, tokens_used
return content, "llm_rewritten", None, tokens_used
@app.get("/healthz")
def healthz() -> Dict[str, Any]:
@@ -242,6 +486,10 @@ def create_app(phrases_path: Optional[Path] = None) -> FastAPI:
"phrases_loaded": app.state.phrase_total,
"puzzles_loaded": len(app.state.phrase_bank),
"phrases_path": app.state.phrases_path,
"safety_puzzles_loaded": len(app.state.safety_bank),
"safety_path": app.state.safety_path,
"litellm_url": app.state.litellm_url,
"llm_model": app.state.llm_model,
}
@app.get("/hints/coverage")
@@ -249,33 +497,47 @@ def create_app(phrases_path: Optional[Path] = None) -> FastAPI:
return app.state.coverage
@app.post("/hints/ask", response_model=HintAskResponse)
def hints_ask(payload: HintAskRequest, request: Request) -> HintAskResponse:
async def hints_ask(
payload: HintAskRequest,
request: Request,
rewrite: bool = Query(True, description="Set false to bypass LLM and return verbatim static phrase"),
) -> HintAskResponse:
started = time.perf_counter()
status_code = 200
source = "static"
model_used = "none"
safety_triggered: Optional[str] = None
tokens_used: Optional[int] = None
hint_static = ""
hint_rewritten: Optional[str] = None
try:
puzzle = app.state.phrase_bank.get(payload.puzzle_id)
if puzzle is None:
status_code = 404
raise HTTPException(
status_code=404,
detail=f"unknown puzzle_id: {payload.puzzle_id!r}",
hint_static = _resolve_static(payload.puzzle_id, payload.level)
if not rewrite:
source = "static"
model_used = "none"
final_hint = hint_static
else:
rewritten, source, safety_triggered, tokens_used = await _try_rewrite(
hint_static, payload.puzzle_id, payload.level
)
level_key = f"level_{payload.level}"
phrases = puzzle.get(level_key) or []
if not phrases:
status_code = 404
raise HTTPException(
status_code=404,
detail=(
f"no phrases for {payload.puzzle_id} at {level_key} "
f"(available: {[k for k, v in puzzle.items() if v]})"
),
)
hint = random.choice(phrases)
if rewritten is not None:
hint_rewritten = rewritten
final_hint = rewritten
model_used = app.state.llm_model
else:
final_hint = hint_static
model_used = "none"
latency_ms = round((time.perf_counter() - started) * 1000, 2)
return HintAskResponse(
hint=hint,
hint=final_hint,
hint_static=hint_static,
hint_rewritten=hint_rewritten,
level=payload.level,
puzzle_id=payload.puzzle_id,
source=source,
model_used=model_used,
latency_ms=latency_ms,
)
except HTTPException as exc:
status_code = exc.status_code
@@ -288,6 +550,75 @@ def create_app(phrases_path: Optional[Path] = None) -> FastAPI:
"level": payload.level,
"session_id": payload.session_id,
"client": request.client.host if request.client else None,
"rewrite_requested": rewrite,
"source": source,
"model_used": model_used,
"tokens_used": tokens_used,
"safety_triggered": safety_triggered,
"latency_ms": round((time.perf_counter() - started) * 1000, 2),
"status": status_code,
})
@app.post("/hints/rewrite", response_model=HintRewriteResponse)
async def hints_rewrite(
payload: HintRewriteRequest, request: Request,
) -> HintRewriteResponse:
"""Debug-only path: forces the LLM rewrite and returns it.
Differs from `/hints/ask` in that it surfaces a 502 if the rewrite
fails (no silent verbatim fallback). Useful for prompt tuning.
"""
started = time.perf_counter()
status_code = 200
source = "static"
safety_triggered: Optional[str] = None
tokens_used: Optional[int] = None
try:
hint_static = _resolve_static(payload.puzzle_id, payload.level)
rewritten, source, safety_triggered, tokens_used = await _try_rewrite(
hint_static,
payload.puzzle_id,
payload.level,
max_tokens=payload.max_tokens or DEFAULT_LLM_MAX_TOKENS,
temperature=(
payload.temperature
if payload.temperature is not None
else DEFAULT_LLM_TEMPERATURE
),
)
if rewritten is None:
status_code = 502
detail = (
"rewrite blocked by safety filter"
if safety_triggered
else "rewrite failed (timeout/backend error)"
)
raise HTTPException(status_code=502, detail=detail)
latency_ms = round((time.perf_counter() - started) * 1000, 2)
return HintRewriteResponse(
hint_static=hint_static,
hint_rewritten=rewritten,
level=payload.level,
puzzle_id=payload.puzzle_id,
model_used=app.state.llm_model,
latency_ms=latency_ms,
tokens_used=tokens_used,
)
except HTTPException as exc:
status_code = exc.status_code
raise
finally:
_emit_log({
"ts": time.time(),
"event": "hints_rewrite",
"puzzle_id": payload.puzzle_id,
"level": payload.level,
"session_id": payload.session_id,
"client": request.client.host if request.client else None,
"source": source,
"model_used": app.state.llm_model,
"tokens_used": tokens_used,
"safety_triggered": safety_triggered,
"latency_ms": round((time.perf_counter() - started) * 1000, 2),
"status": status_code,
})