feat(hints): add LLM rewrite via LiteLLM hints-deep
This commit is contained in:
@@ -71,7 +71,7 @@ playtest:
|
||||
--snapshot game/scenarios/playtests/snapshots/zacus_v3_60min_tech.snapshot.json
|
||||
|
||||
hints-serve:
|
||||
uv run --with fastapi --with uvicorn --with pyyaml --with pydantic \
|
||||
uv run --with fastapi --with uvicorn --with pyyaml --with pydantic --with httpx \
|
||||
uvicorn tools.hints.server:app --reload --port 8300
|
||||
|
||||
hints-test:
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# Hints safety filter — banned words/numbers per puzzle.
|
||||
#
|
||||
# When the LLM rewrites a hint (Phase P2 of hints engine), its output is
|
||||
# scanned for any token in the per-puzzle banned list (case-insensitive,
|
||||
# accent-insensitive). If a hit is detected, the rewrite is rejected and
|
||||
# the engine falls back to the verbatim base phrase.
|
||||
#
|
||||
# The intent is to prevent the LLM from accidentally leaking the literal
|
||||
# solution while it is rewording a level-1 or level-2 hint. Bans here are
|
||||
# intentionally narrow — only the *answer* tokens, not the puzzle vocabulary.
|
||||
# Default (no entry for a puzzle): no filtering.
|
||||
#
|
||||
# Spec: docs/superpowers/specs/2026-05-03-hints-engine-design.md §4 ("Safety rail")
|
||||
|
||||
banned_per_puzzle:
|
||||
# Sound puzzle: 5-button colour sequence Rouge/Bleu/Jaune/Rouge/Vert.
|
||||
# The full ordered solution lives at level_3; we forbid the rewriter from
|
||||
# echoing the full sequence at level_1 or level_2.
|
||||
P1_SON:
|
||||
- "rouge bleu jaune"
|
||||
- "bleu jaune rouge vert"
|
||||
|
||||
# QR cache hunt: the level_3 hint enumerates the 6-cache scan order.
|
||||
# Any of these landmark words leaking would skip the deduction.
|
||||
P3_QR:
|
||||
- "etagere gauche"
|
||||
- "tiroir rouge"
|
||||
- "boite bleue"
|
||||
- "box-3"
|
||||
|
||||
# Radio frequency: the answer is literally 1337 Hz ("LEET").
|
||||
P4_RADIO:
|
||||
- "1337"
|
||||
- "leet"
|
||||
|
||||
# Morse code: the message is "ZACUS". The level_3 hint also contains the
|
||||
# raw dot/dash transcription. Rewriter must not pre-spell either.
|
||||
P5_MORSE:
|
||||
- "zacus"
|
||||
- "z=--.."
|
||||
- "tiret-tiret-point"
|
||||
+207
-20
@@ -1,15 +1,18 @@
|
||||
"""Tests for tools.hints.server (P1 — static phrase lookup).
|
||||
"""Tests for tools.hints.server (P1 static lookup + P2 LLM rewrite).
|
||||
|
||||
Run via:
|
||||
make hints-test
|
||||
or:
|
||||
uv run --with fastapi --with uvicorn --with pyyaml --with pydantic \
|
||||
--with pytest --with httpx pytest tests/hints/ -v
|
||||
--with pytest --with httpx --with pytest-asyncio \
|
||||
pytest tests/hints/ -v
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
@@ -17,10 +20,13 @@ from fastapi.testclient import TestClient
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from tools.hints.server import ( # noqa: E402 (import after sys.path tweak)
|
||||
from tools.hints import server as server_module # noqa: E402
|
||||
from tools.hints.server import ( # noqa: E402
|
||||
DEFAULT_PHRASES_PATH,
|
||||
DEFAULT_SAFETY_PATH,
|
||||
create_app,
|
||||
load_phrase_bank,
|
||||
safety_check,
|
||||
)
|
||||
|
||||
|
||||
@@ -31,7 +37,7 @@ from tools.hints.server import ( # noqa: E402 (import after sys.path tweak)
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def app():
|
||||
return create_app(DEFAULT_PHRASES_PATH)
|
||||
return create_app(DEFAULT_PHRASES_PATH, DEFAULT_SAFETY_PATH)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
@@ -43,8 +49,7 @@ def client(app):
|
||||
def real_puzzle_id():
|
||||
"""First puzzle in the bank that has a non-empty level_1.
|
||||
|
||||
Using a real puzzle from the actual YAML keeps the test honest — if
|
||||
`npc_phrases.yaml` drifts, the suite catches it.
|
||||
Using a real puzzle from the actual YAML keeps the test honest.
|
||||
"""
|
||||
bank = load_phrase_bank(DEFAULT_PHRASES_PATH)
|
||||
for pid, levels in bank.items():
|
||||
@@ -53,8 +58,31 @@ def real_puzzle_id():
|
||||
pytest.fail("no puzzle with level_1 found in npc_phrases.yaml")
|
||||
|
||||
|
||||
def _make_fake_call(
|
||||
content: str = "Mon cher, écoutez le rythme et comptez les battements.",
|
||||
*,
|
||||
raise_exc: Exception | None = None,
|
||||
delay_s: float = 0.0,
|
||||
tokens_used: int = 42,
|
||||
):
|
||||
"""Build a coroutine that mimics `_call_litellm` for monkeypatching."""
|
||||
|
||||
async def fake(**kwargs: Any) -> Tuple[str, Dict[str, Any]]:
|
||||
if delay_s:
|
||||
await asyncio.sleep(delay_s)
|
||||
if raise_exc is not None:
|
||||
raise raise_exc
|
||||
raw = {
|
||||
"choices": [{"message": {"content": content}}],
|
||||
"usage": {"total_tokens": tokens_used},
|
||||
}
|
||||
return content, raw
|
||||
|
||||
return fake
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Health + coverage
|
||||
# Health + coverage (P1 + P2 health surface)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@@ -66,6 +94,10 @@ def test_healthz_returns_ok_with_counts(client):
|
||||
assert body["phrases_loaded"] > 0
|
||||
assert body["puzzles_loaded"] > 0
|
||||
assert body["phrases_path"].endswith("npc_phrases.yaml")
|
||||
# P2 fields
|
||||
assert "safety_puzzles_loaded" in body
|
||||
assert "litellm_url" in body
|
||||
assert body["llm_model"]
|
||||
|
||||
|
||||
def test_coverage_summary_shape_is_complete(client):
|
||||
@@ -73,17 +105,14 @@ def test_coverage_summary_shape_is_complete(client):
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
|
||||
# Required top-level fields
|
||||
for key in ("total_puzzles", "puzzles_by_level_count",
|
||||
"percent_by_level_count", "per_puzzle"):
|
||||
assert key in body, f"coverage missing field {key!r}"
|
||||
|
||||
# Bucket counts must sum to total
|
||||
bucket_sum = sum(body["puzzles_by_level_count"].values())
|
||||
assert bucket_sum == body["total_puzzles"]
|
||||
assert body["total_puzzles"] >= 1
|
||||
|
||||
# Each per_puzzle row has the expected sub-fields
|
||||
for row in body["per_puzzle"]:
|
||||
assert "puzzle_id" in row
|
||||
assert "levels_present" in row
|
||||
@@ -93,13 +122,13 @@ def test_coverage_summary_shape_is_complete(client):
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /hints/ask happy path + errors
|
||||
# /hints/ask happy path + errors (verbatim path with ?rewrite=false)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_hints_ask_happy_path_returns_real_phrase(client, real_puzzle_id):
|
||||
resp = client.post(
|
||||
"/hints/ask",
|
||||
"/hints/ask?rewrite=false",
|
||||
json={"puzzle_id": real_puzzle_id, "level": 1, "session_id": "test-session"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
@@ -107,18 +136,19 @@ def test_hints_ask_happy_path_returns_real_phrase(client, real_puzzle_id):
|
||||
assert body["puzzle_id"] == real_puzzle_id
|
||||
assert body["level"] == 1
|
||||
assert body["source"] == "static"
|
||||
assert body["model_used"] == "none"
|
||||
assert body["hint_rewritten"] is None
|
||||
assert isinstance(body["hint"], str)
|
||||
assert len(body["hint"]) > 0
|
||||
assert body["hint"] == body["hint_static"]
|
||||
|
||||
# The phrase must be one of the real phrases in the bank — guards
|
||||
# against accidental string mangling in the loader.
|
||||
bank = load_phrase_bank(DEFAULT_PHRASES_PATH)
|
||||
assert body["hint"] in bank[real_puzzle_id]["level_1"]
|
||||
|
||||
|
||||
def test_hints_ask_unknown_puzzle_returns_404(client):
|
||||
resp = client.post(
|
||||
"/hints/ask",
|
||||
"/hints/ask?rewrite=false",
|
||||
json={"puzzle_id": "P999_DOES_NOT_EXIST", "level": 1},
|
||||
)
|
||||
assert resp.status_code == 404
|
||||
@@ -126,16 +156,15 @@ def test_hints_ask_unknown_puzzle_returns_404(client):
|
||||
|
||||
|
||||
def test_hints_ask_invalid_level_returns_422(client, real_puzzle_id):
|
||||
# Pydantic validation rejects out-of-range level → 422 (Unprocessable)
|
||||
resp = client.post(
|
||||
"/hints/ask",
|
||||
"/hints/ask?rewrite=false",
|
||||
json={"puzzle_id": real_puzzle_id, "level": 7},
|
||||
)
|
||||
assert resp.status_code == 422
|
||||
|
||||
|
||||
def test_hints_ask_preserves_french_diacritics(client, real_puzzle_id):
|
||||
"""UTF-8 round-trip must keep accented characters intact."""
|
||||
"""UTF-8 round-trip must keep accented characters intact (verbatim path)."""
|
||||
bank = load_phrase_bank(DEFAULT_PHRASES_PATH)
|
||||
has_accent = any(
|
||||
any(ch in phrase for ch in "éèêàùôîçÉÈÊÀÙÔÎÇ")
|
||||
@@ -145,14 +174,172 @@ def test_hints_ask_preserves_french_diacritics(client, real_puzzle_id):
|
||||
if not has_accent:
|
||||
pytest.skip(f"{real_puzzle_id} has no accented phrases — skipping diacritic check")
|
||||
|
||||
# Try several requests — at least one should land on an accented phrase
|
||||
found_accent = False
|
||||
for _ in range(20):
|
||||
resp = client.post(
|
||||
"/hints/ask", json={"puzzle_id": real_puzzle_id, "level": 1}
|
||||
"/hints/ask?rewrite=false", json={"puzzle_id": real_puzzle_id, "level": 1}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
if any(ch in resp.json()["hint"] for ch in "éèêàùôîçÉÈÊÀÙÔÎÇ"):
|
||||
found_accent = True
|
||||
break
|
||||
assert found_accent, "diacritics never round-tripped — encoding bug?"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Safety filter unit
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_safety_check_is_accent_and_case_insensitive():
|
||||
banned = ["Étagère gauche", "1337"]
|
||||
assert safety_check("rien à signaler", banned) is None
|
||||
assert safety_check("vise l'ETAGERE GAUCHE rapidement", banned) == "Étagère gauche"
|
||||
assert safety_check("la fréquence est 1337 hertz", banned) == "1337"
|
||||
assert safety_check("anything", []) is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /hints/ask P2 — LLM rewrite paths (mocked LiteLLM)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_ask_with_rewrite_uses_llm(client, real_puzzle_id, monkeypatch):
|
||||
"""Rewrite path: mocked LLM returns clean content → source=llm_rewritten."""
|
||||
fake = _make_fake_call(
|
||||
content="Ah, mon cher disciple ! Écoutez ce rythme avec ferveur.",
|
||||
tokens_used=37,
|
||||
)
|
||||
monkeypatch.setattr(server_module, "_call_litellm", fake)
|
||||
|
||||
resp = client.post(
|
||||
"/hints/ask",
|
||||
json={"puzzle_id": real_puzzle_id, "level": 1, "session_id": "rw-test"},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body["source"] == "llm_rewritten"
|
||||
assert body["model_used"] == client.app.state.llm_model
|
||||
assert body["hint_rewritten"] == "Ah, mon cher disciple ! Écoutez ce rythme avec ferveur."
|
||||
assert body["hint"] == body["hint_rewritten"]
|
||||
assert isinstance(body["hint_static"], str) and len(body["hint_static"]) > 0
|
||||
assert body["hint_static"] != body["hint_rewritten"]
|
||||
|
||||
|
||||
def test_ask_rewrite_fallback_on_timeout(client, real_puzzle_id, monkeypatch):
|
||||
"""asyncio.TimeoutError from `_call_litellm` → verbatim fallback."""
|
||||
fake = _make_fake_call(raise_exc=asyncio.TimeoutError())
|
||||
monkeypatch.setattr(server_module, "_call_litellm", fake)
|
||||
|
||||
resp = client.post(
|
||||
"/hints/ask",
|
||||
json={"puzzle_id": real_puzzle_id, "level": 1},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["source"] == "llm_fallback_static"
|
||||
assert body["model_used"] == "none"
|
||||
assert body["hint_rewritten"] is None
|
||||
assert body["hint"] == body["hint_static"]
|
||||
|
||||
|
||||
def test_ask_rewrite_fallback_on_safety_trigger(client, monkeypatch):
|
||||
"""LLM emits a banned word → safety filter trips → verbatim fallback."""
|
||||
# P4_RADIO bans "1337" — make the LLM "leak" it.
|
||||
fake = _make_fake_call(
|
||||
content="Tournez le bouton jusqu'à ce que l'écran affiche 1337 hertz précisément.",
|
||||
)
|
||||
monkeypatch.setattr(server_module, "_call_litellm", fake)
|
||||
|
||||
resp = client.post(
|
||||
"/hints/ask",
|
||||
json={"puzzle_id": "P4_RADIO", "level": 1},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["source"] == "llm_fallback_static"
|
||||
assert body["model_used"] == "none"
|
||||
assert body["hint_rewritten"] is None
|
||||
assert body["hint"] == body["hint_static"]
|
||||
# Sanity: the static phrase itself isn't the leaky one (it stays in bank as-is)
|
||||
assert isinstance(body["hint_static"], str) and len(body["hint_static"]) > 0
|
||||
|
||||
|
||||
def test_ask_query_rewrite_false_bypasses_llm(client, real_puzzle_id, monkeypatch):
|
||||
"""`?rewrite=false` must short-circuit and never call _call_litellm."""
|
||||
called = {"n": 0}
|
||||
|
||||
async def explode(**kwargs: Any):
|
||||
called["n"] += 1
|
||||
raise AssertionError("LLM was called despite rewrite=false")
|
||||
|
||||
monkeypatch.setattr(server_module, "_call_litellm", explode)
|
||||
|
||||
resp = client.post(
|
||||
"/hints/ask?rewrite=false",
|
||||
json={"puzzle_id": real_puzzle_id, "level": 1},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert called["n"] == 0
|
||||
assert body["source"] == "static"
|
||||
assert body["model_used"] == "none"
|
||||
assert body["hint_rewritten"] is None
|
||||
|
||||
|
||||
def test_ask_rewrite_fallback_on_backend_error(client, real_puzzle_id, monkeypatch):
|
||||
"""Generic backend RuntimeError (HTTP 500, malformed JSON…) → fallback."""
|
||||
fake = _make_fake_call(raise_exc=RuntimeError("litellm http 500: oops"))
|
||||
monkeypatch.setattr(server_module, "_call_litellm", fake)
|
||||
|
||||
resp = client.post(
|
||||
"/hints/ask",
|
||||
json={"puzzle_id": real_puzzle_id, "level": 1},
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
body = resp.json()
|
||||
assert body["source"] == "llm_fallback_static"
|
||||
assert body["hint"] == body["hint_static"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# /hints/rewrite debug endpoint
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_rewrite_endpoint_works(client, real_puzzle_id, monkeypatch):
|
||||
fake = _make_fake_call(
|
||||
content="Mon cher, ne sous-estimez jamais la mémoire d'une mélodie ancienne.",
|
||||
tokens_used=29,
|
||||
)
|
||||
monkeypatch.setattr(server_module, "_call_litellm", fake)
|
||||
|
||||
resp = client.post(
|
||||
"/hints/rewrite",
|
||||
json={
|
||||
"puzzle_id": real_puzzle_id,
|
||||
"level": 1,
|
||||
"max_tokens": 64,
|
||||
"temperature": 0.5,
|
||||
},
|
||||
)
|
||||
assert resp.status_code == 200, resp.text
|
||||
body = resp.json()
|
||||
assert body["puzzle_id"] == real_puzzle_id
|
||||
assert body["level"] == 1
|
||||
assert body["model_used"] == client.app.state.llm_model
|
||||
assert body["hint_rewritten"].startswith("Mon cher")
|
||||
assert body["tokens_used"] == 29
|
||||
assert isinstance(body["hint_static"], str) and len(body["hint_static"]) > 0
|
||||
|
||||
|
||||
def test_rewrite_endpoint_502_on_safety_block(client, monkeypatch):
|
||||
fake = _make_fake_call(content="La fréquence est 1337 hertz, sans hésitation.")
|
||||
monkeypatch.setattr(server_module, "_call_litellm", fake)
|
||||
|
||||
resp = client.post(
|
||||
"/hints/rewrite",
|
||||
json={"puzzle_id": "P4_RADIO", "level": 1},
|
||||
)
|
||||
assert resp.status_code == 502
|
||||
assert "safety" in resp.json()["detail"].lower()
|
||||
|
||||
+384
-53
@@ -1,21 +1,37 @@
|
||||
"""Hints engine — Phase P1 FastAPI server.
|
||||
"""Hints engine — Phase P2 FastAPI server.
|
||||
|
||||
Static phrase lookup (P1) + LLM rewrite layer via LiteLLM `hints-deep` (P2).
|
||||
The rewriter restyles the static base phrase in the Professor Zacus voice
|
||||
without inventing new content. Safety filter blocks rewrites that leak
|
||||
solution tokens. On timeout / error / safety trip, the engine falls back
|
||||
to the verbatim static phrase so the game never blocks.
|
||||
|
||||
Stateless static phrase lookup from `game/scenarios/npc_phrases.yaml`.
|
||||
Endpoints:
|
||||
- GET /healthz — liveness probe + phrases-loaded count
|
||||
- POST /hints/ask — return base phrase for {puzzle_id, level}
|
||||
- GET /hints/coverage — per-puzzle level coverage audit
|
||||
- GET /healthz — liveness probe + phrases-loaded count
|
||||
- POST /hints/ask — return base phrase, optionally LLM-rewritten
|
||||
(?rewrite=false to bypass LLM)
|
||||
- POST /hints/rewrite — debug: rewrite-only path, no static fallback
|
||||
- GET /hints/coverage — per-puzzle level coverage audit
|
||||
|
||||
Run:
|
||||
uv run --with fastapi --with uvicorn --with pyyaml --with pydantic \
|
||||
--with httpx \
|
||||
uvicorn tools.hints.server:app --reload --port 8300
|
||||
|
||||
Spec: docs/superpowers/specs/2026-05-03-hints-engine-design.md
|
||||
P1 scope is intentionally narrow: no LLM rewrite, no anti-cheat,
|
||||
no adaptive level selection, no auto-trigger. Those land in P2-P6.
|
||||
Env:
|
||||
LITELLM_URL (default http://192.168.0.120:4000)
|
||||
LITELLM_MASTER_KEY (default sk-zacus-local-dev-do-not-share)
|
||||
HINTS_LLM_MODEL (default hints-deep)
|
||||
HINTS_LLM_TIMEOUT_S (default 8.0)
|
||||
HINTS_SAFETY_PATH (default game/scenarios/hints_safety.yaml)
|
||||
ZACUS_NPC_PHRASES_PATH (default game/scenarios/npc_phrases.yaml)
|
||||
|
||||
Spec: docs/superpowers/specs/2026-05-03-hints-engine-design.md §4
|
||||
P3 (anti-cheat) and P4 (adaptive level) are intentionally NOT in scope here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -23,11 +39,13 @@ import random
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import unicodedata
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi import FastAPI, HTTPException, Query, Request
|
||||
from pydantic import BaseModel, Field, conint
|
||||
|
||||
|
||||
@@ -37,15 +55,26 @@ from pydantic import BaseModel, Field, conint
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
DEFAULT_PHRASES_PATH = REPO_ROOT / "game" / "scenarios" / "npc_phrases.yaml"
|
||||
DEFAULT_SAFETY_PATH = REPO_ROOT / "game" / "scenarios" / "hints_safety.yaml"
|
||||
PHRASES_PATH_ENV = "ZACUS_NPC_PHRASES_PATH"
|
||||
SAFETY_PATH_ENV = "HINTS_SAFETY_PATH"
|
||||
|
||||
DEFAULT_LITELLM_URL = "http://192.168.0.120:4000"
|
||||
DEFAULT_LITELLM_KEY = "sk-zacus-local-dev-do-not-share"
|
||||
DEFAULT_LLM_MODEL = "hints-deep"
|
||||
DEFAULT_LLM_TIMEOUT_S = 8.0
|
||||
DEFAULT_LLM_MAX_TOKENS = 80
|
||||
DEFAULT_LLM_TEMPERATURE = 0.4
|
||||
|
||||
ZACUS_SYSTEM_PROMPT = (
|
||||
"Tu es le Professeur Zacus, savant excentrique. Réécris l'indice "
|
||||
"ci-dessous en restant fidèle à son contenu mais avec ton style "
|
||||
"théâtral et un ton dramatique adapté à un escape room. 2 phrases "
|
||||
"max. Ne révèle pas la solution, garde les nombres, codes et noms "
|
||||
"propres exacts. Pas d'emoji."
|
||||
)
|
||||
|
||||
# Puzzle IDs that look like V3 numbered puzzles (P1_FOO, P12_BAR, ...).
|
||||
# Authoring drift: `P1_SON`..`P7_COFFRE` are nested under `endings:` in the
|
||||
# current YAML (indentation bug — see audit). We surface them anyway because
|
||||
# their internal `key:` values advertise `hints.<id>.level_N`. Fix later in
|
||||
# `npc_phrases.yaml`; do NOT fix here (P1 instructions forbid YAML edits).
|
||||
PUZZLE_ID_PATTERN = re.compile(r"^P\d+_[A-Z0-9_]+$")
|
||||
|
||||
ALL_LEVELS = (1, 2, 3)
|
||||
|
||||
LOG = logging.getLogger("hints.server")
|
||||
@@ -80,7 +109,6 @@ def _normalize_level_block(block: Any) -> List[str]:
|
||||
if isinstance(block, str):
|
||||
return [block.strip()] if block.strip() else []
|
||||
if isinstance(block, dict):
|
||||
# Tolerate `{text: "..."}` at the level position
|
||||
text = _extract_text(block).strip()
|
||||
return [text] if text else []
|
||||
return []
|
||||
@@ -95,10 +123,7 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
|
||||
2. `endings.<puzzle_id>.level_N` where puzzle_id matches PUZZLE_ID_PATTERN
|
||||
(the V3 P1-P7 hints currently mis-nested under `endings:`)
|
||||
|
||||
Raises:
|
||||
FileNotFoundError if `path` doesn't exist.
|
||||
ValueError if the YAML lacks a top-level `hints` mapping or no usable
|
||||
puzzle entries are found.
|
||||
Raises FileNotFoundError or ValueError on schema problems.
|
||||
"""
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"npc_phrases.yaml not found at {path}")
|
||||
@@ -115,7 +140,6 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
|
||||
|
||||
bank: Dict[str, Dict[str, List[str]]] = {}
|
||||
|
||||
# 1. Canonical hints.* puzzles
|
||||
for puzzle_id, levels in hints_section.items():
|
||||
if not isinstance(levels, dict):
|
||||
continue
|
||||
@@ -124,7 +148,6 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
|
||||
for n in ALL_LEVELS
|
||||
}
|
||||
|
||||
# 2. Mis-nested V3 puzzles under endings.*
|
||||
endings_section = data.get("endings")
|
||||
if isinstance(endings_section, dict):
|
||||
for puzzle_id, levels in endings_section.items():
|
||||
@@ -133,7 +156,7 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
|
||||
if not isinstance(levels, dict):
|
||||
continue
|
||||
if puzzle_id in bank:
|
||||
continue # canonical hints.* wins
|
||||
continue
|
||||
bank[puzzle_id] = {
|
||||
f"level_{n}": _normalize_level_block(levels.get(f"level_{n}"))
|
||||
for n in ALL_LEVELS
|
||||
@@ -148,6 +171,32 @@ def load_phrase_bank(path: Path) -> Dict[str, Dict[str, List[str]]]:
|
||||
return bank
|
||||
|
||||
|
||||
def load_safety_bank(path: Path) -> Dict[str, List[str]]:
|
||||
"""Load `hints_safety.yaml` and return `{puzzle_id: [banned_word, ...]}`.
|
||||
|
||||
Empty mapping if the file is absent (P2 default, no filtering anywhere).
|
||||
Banned words are stored as-is; comparison is normalized at filter time.
|
||||
"""
|
||||
if not path.is_file():
|
||||
LOG.info("hints_safety.yaml not found at %s — running with no safety filter", path)
|
||||
return {}
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = yaml.safe_load(f) or {}
|
||||
if not isinstance(data, dict):
|
||||
return {}
|
||||
raw = data.get("banned_per_puzzle") or {}
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
out: Dict[str, List[str]] = {}
|
||||
for puzzle_id, words in raw.items():
|
||||
if not isinstance(words, list):
|
||||
continue
|
||||
cleaned = [str(w).strip() for w in words if isinstance(w, (str, int, float)) and str(w).strip()]
|
||||
if cleaned:
|
||||
out[str(puzzle_id)] = cleaned
|
||||
return out
|
||||
|
||||
|
||||
def compute_coverage(bank: Dict[str, Dict[str, List[str]]]) -> Dict[str, Any]:
|
||||
"""Build the audit summary returned by GET /hints/coverage."""
|
||||
per_puzzle: List[Dict[str, Any]] = []
|
||||
@@ -182,6 +231,89 @@ def count_phrases(bank: Dict[str, Dict[str, List[str]]]) -> int:
|
||||
return sum(len(level) for puzzle in bank.values() for level in puzzle.values())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Safety filter
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _strip_accents(s: str) -> str:
|
||||
"""Lower + strip diacritics for case/accent-insensitive matching."""
|
||||
nfkd = unicodedata.normalize("NFKD", s)
|
||||
return "".join(ch for ch in nfkd if not unicodedata.combining(ch)).lower()
|
||||
|
||||
|
||||
def safety_check(text: str, banned_words: List[str]) -> Optional[str]:
|
||||
"""Return the first matched banned word if `text` contains one, else None.
|
||||
|
||||
Matching is case- and accent-insensitive on a normalized substring.
|
||||
Empty `banned_words` means "no filter" → always None.
|
||||
"""
|
||||
if not banned_words:
|
||||
return None
|
||||
haystack = _strip_accents(text)
|
||||
for word in banned_words:
|
||||
needle = _strip_accents(word)
|
||||
if needle and needle in haystack:
|
||||
return word
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# LiteLLM call (mockable in tests via monkeypatch)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_messages(hint_static: str, puzzle_id: str, level: int) -> List[Dict[str, str]]:
|
||||
user_prompt = (
|
||||
f"Niveau {level}, énigme {puzzle_id}. "
|
||||
f"Phrase originale à reformuler : « {hint_static} »"
|
||||
)
|
||||
return [
|
||||
{"role": "system", "content": ZACUS_SYSTEM_PROMPT},
|
||||
{"role": "user", "content": user_prompt},
|
||||
]
|
||||
|
||||
|
||||
async def _call_litellm(
|
||||
*,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
model: str,
|
||||
messages: List[Dict[str, str]],
|
||||
timeout_s: float,
|
||||
max_tokens: int,
|
||||
temperature: float,
|
||||
) -> Tuple[str, Dict[str, Any]]:
|
||||
"""POST to LiteLLM `/v1/chat/completions`, return (content, raw_json).
|
||||
|
||||
Raises `asyncio.TimeoutError`, `httpx.HTTPError`, or `RuntimeError` on
|
||||
a non-200 / malformed response. Callers MUST catch and fall back.
|
||||
Tests monkeypatch this whole function to avoid network I/O.
|
||||
"""
|
||||
payload = {
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
}
|
||||
async with httpx.AsyncClient(timeout=timeout_s) as client:
|
||||
resp = await client.post(
|
||||
f"{base_url}/v1/chat/completions",
|
||||
headers={"Authorization": f"Bearer {api_key}"},
|
||||
json=payload,
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"litellm http {resp.status_code}: {resp.text[:200]}")
|
||||
data = resp.json()
|
||||
try:
|
||||
content = data["choices"][0]["message"]["content"].strip()
|
||||
except (KeyError, IndexError, TypeError) as exc:
|
||||
raise RuntimeError(f"litellm malformed response: {exc}") from exc
|
||||
if not content:
|
||||
raise RuntimeError("litellm returned empty content")
|
||||
return content, data
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pydantic request/response models
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -195,9 +327,31 @@ class HintAskRequest(BaseModel):
|
||||
|
||||
class HintAskResponse(BaseModel):
|
||||
hint: str
|
||||
hint_static: str
|
||||
hint_rewritten: Optional[str] = None
|
||||
level: int
|
||||
puzzle_id: str
|
||||
source: str = "static"
|
||||
source: str # "static" | "llm_rewritten" | "llm_fallback_static"
|
||||
model_used: str # "hints-deep" | "none"
|
||||
latency_ms: float
|
||||
|
||||
|
||||
class HintRewriteRequest(BaseModel):
|
||||
puzzle_id: str = Field(..., min_length=1)
|
||||
level: conint(ge=1, le=3) # type: ignore[valid-type]
|
||||
session_id: Optional[str] = None
|
||||
max_tokens: Optional[conint(ge=8, le=512)] = None # type: ignore[valid-type]
|
||||
temperature: Optional[float] = Field(None, ge=0.0, le=2.0)
|
||||
|
||||
|
||||
class HintRewriteResponse(BaseModel):
|
||||
hint_static: str
|
||||
hint_rewritten: str
|
||||
level: int
|
||||
puzzle_id: str
|
||||
model_used: str
|
||||
latency_ms: float
|
||||
tokens_used: Optional[int] = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -210,30 +364,120 @@ def _resolve_phrases_path() -> Path:
|
||||
return Path(override).resolve() if override else DEFAULT_PHRASES_PATH
|
||||
|
||||
|
||||
def _resolve_safety_path() -> Path:
|
||||
override = os.environ.get(SAFETY_PATH_ENV)
|
||||
return Path(override).resolve() if override else DEFAULT_SAFETY_PATH
|
||||
|
||||
|
||||
def _emit_log(record: Dict[str, Any]) -> None:
|
||||
"""Structured JSON log line on stdout (one dict per request)."""
|
||||
sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def create_app(phrases_path: Optional[Path] = None) -> FastAPI:
|
||||
"""Build the FastAPI app. Pass `phrases_path` for tests, default for prod."""
|
||||
path = phrases_path or _resolve_phrases_path()
|
||||
bank = load_phrase_bank(path) # raises if cracked → uvicorn refuses to start
|
||||
def create_app(
|
||||
phrases_path: Optional[Path] = None,
|
||||
safety_path: Optional[Path] = None,
|
||||
) -> FastAPI:
|
||||
"""Build the FastAPI app. Pass paths for tests, defaults for prod."""
|
||||
p_path = phrases_path or _resolve_phrases_path()
|
||||
s_path = safety_path or _resolve_safety_path()
|
||||
bank = load_phrase_bank(p_path)
|
||||
safety = load_safety_bank(s_path)
|
||||
coverage_cache = compute_coverage(bank)
|
||||
phrase_total = count_phrases(bank)
|
||||
|
||||
app = FastAPI(
|
||||
title="Zacus Hints Engine",
|
||||
version="0.1.0-P1",
|
||||
description="Static NPC hint phrase lookup. Phase P1 of the hints engine spec.",
|
||||
version="0.2.0-P2",
|
||||
description=(
|
||||
"Static NPC hint phrase lookup + LLM rewrite via LiteLLM hints-deep. "
|
||||
"Phase P2 of the hints engine spec."
|
||||
),
|
||||
)
|
||||
|
||||
# Stash for handlers + tests
|
||||
app.state.phrase_bank = bank
|
||||
app.state.safety_bank = safety
|
||||
app.state.coverage = coverage_cache
|
||||
app.state.phrase_total = phrase_total
|
||||
app.state.phrases_path = str(path)
|
||||
app.state.phrases_path = str(p_path)
|
||||
app.state.safety_path = str(s_path)
|
||||
app.state.litellm_url = os.environ.get("LITELLM_URL", DEFAULT_LITELLM_URL)
|
||||
app.state.litellm_key = os.environ.get("LITELLM_MASTER_KEY", DEFAULT_LITELLM_KEY)
|
||||
app.state.llm_model = os.environ.get("HINTS_LLM_MODEL", DEFAULT_LLM_MODEL)
|
||||
app.state.llm_timeout_s = float(
|
||||
os.environ.get("HINTS_LLM_TIMEOUT_S", str(DEFAULT_LLM_TIMEOUT_S))
|
||||
)
|
||||
|
||||
def _resolve_static(puzzle_id: str, level: int) -> str:
|
||||
puzzle = app.state.phrase_bank.get(puzzle_id)
|
||||
if puzzle is None:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"unknown puzzle_id: {puzzle_id!r}",
|
||||
)
|
||||
level_key = f"level_{level}"
|
||||
phrases = puzzle.get(level_key) or []
|
||||
if not phrases:
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=(
|
||||
f"no phrases for {puzzle_id} at {level_key} "
|
||||
f"(available: {[k for k, v in puzzle.items() if v]})"
|
||||
),
|
||||
)
|
||||
return random.choice(phrases)
|
||||
|
||||
async def _try_rewrite(
|
||||
hint_static: str,
|
||||
puzzle_id: str,
|
||||
level: int,
|
||||
*,
|
||||
max_tokens: int = DEFAULT_LLM_MAX_TOKENS,
|
||||
temperature: float = DEFAULT_LLM_TEMPERATURE,
|
||||
) -> Tuple[Optional[str], str, Optional[str], Optional[int]]:
|
||||
"""Best-effort LLM rewrite.
|
||||
|
||||
Returns (rewritten_text_or_none, source_tag, safety_trigger_word_or_none,
|
||||
tokens_used_or_none). `source_tag` is "llm_rewritten" on success and
|
||||
"llm_fallback_static" on any failure (timeout / error / safety trip).
|
||||
"""
|
||||
messages = _build_messages(hint_static, puzzle_id, level)
|
||||
try:
|
||||
content, raw = await asyncio.wait_for(
|
||||
_call_litellm(
|
||||
base_url=app.state.litellm_url,
|
||||
api_key=app.state.litellm_key,
|
||||
model=app.state.llm_model,
|
||||
messages=messages,
|
||||
timeout_s=app.state.llm_timeout_s,
|
||||
max_tokens=max_tokens,
|
||||
temperature=temperature,
|
||||
),
|
||||
timeout=app.state.llm_timeout_s,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
LOG.warning("hints rewrite timeout for %s/level_%d", puzzle_id, level)
|
||||
return None, "llm_fallback_static", None, None
|
||||
except Exception as exc: # noqa: BLE001 fallback covers all backend issues
|
||||
LOG.warning("hints rewrite error for %s/level_%d: %s", puzzle_id, level, exc)
|
||||
return None, "llm_fallback_static", None, None
|
||||
|
||||
tokens_used = None
|
||||
usage = (raw or {}).get("usage") if isinstance(raw, dict) else None
|
||||
if isinstance(usage, dict):
|
||||
tokens_used = usage.get("total_tokens")
|
||||
|
||||
banned = app.state.safety_bank.get(puzzle_id, [])
|
||||
trigger = safety_check(content, banned)
|
||||
if trigger:
|
||||
LOG.warning(
|
||||
"hints safety filter triggered for %s/level_%d on word=%r",
|
||||
puzzle_id, level, trigger,
|
||||
)
|
||||
return None, "llm_fallback_static", trigger, tokens_used
|
||||
|
||||
return content, "llm_rewritten", None, tokens_used
|
||||
|
||||
@app.get("/healthz")
|
||||
def healthz() -> Dict[str, Any]:
|
||||
@@ -242,6 +486,10 @@ def create_app(phrases_path: Optional[Path] = None) -> FastAPI:
|
||||
"phrases_loaded": app.state.phrase_total,
|
||||
"puzzles_loaded": len(app.state.phrase_bank),
|
||||
"phrases_path": app.state.phrases_path,
|
||||
"safety_puzzles_loaded": len(app.state.safety_bank),
|
||||
"safety_path": app.state.safety_path,
|
||||
"litellm_url": app.state.litellm_url,
|
||||
"llm_model": app.state.llm_model,
|
||||
}
|
||||
|
||||
@app.get("/hints/coverage")
|
||||
@@ -249,33 +497,47 @@ def create_app(phrases_path: Optional[Path] = None) -> FastAPI:
|
||||
return app.state.coverage
|
||||
|
||||
@app.post("/hints/ask", response_model=HintAskResponse)
|
||||
def hints_ask(payload: HintAskRequest, request: Request) -> HintAskResponse:
|
||||
async def hints_ask(
|
||||
payload: HintAskRequest,
|
||||
request: Request,
|
||||
rewrite: bool = Query(True, description="Set false to bypass LLM and return verbatim static phrase"),
|
||||
) -> HintAskResponse:
|
||||
started = time.perf_counter()
|
||||
status_code = 200
|
||||
source = "static"
|
||||
model_used = "none"
|
||||
safety_triggered: Optional[str] = None
|
||||
tokens_used: Optional[int] = None
|
||||
hint_static = ""
|
||||
hint_rewritten: Optional[str] = None
|
||||
try:
|
||||
puzzle = app.state.phrase_bank.get(payload.puzzle_id)
|
||||
if puzzle is None:
|
||||
status_code = 404
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"unknown puzzle_id: {payload.puzzle_id!r}",
|
||||
hint_static = _resolve_static(payload.puzzle_id, payload.level)
|
||||
if not rewrite:
|
||||
source = "static"
|
||||
model_used = "none"
|
||||
final_hint = hint_static
|
||||
else:
|
||||
rewritten, source, safety_triggered, tokens_used = await _try_rewrite(
|
||||
hint_static, payload.puzzle_id, payload.level
|
||||
)
|
||||
level_key = f"level_{payload.level}"
|
||||
phrases = puzzle.get(level_key) or []
|
||||
if not phrases:
|
||||
status_code = 404
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=(
|
||||
f"no phrases for {payload.puzzle_id} at {level_key} "
|
||||
f"(available: {[k for k, v in puzzle.items() if v]})"
|
||||
),
|
||||
)
|
||||
hint = random.choice(phrases)
|
||||
if rewritten is not None:
|
||||
hint_rewritten = rewritten
|
||||
final_hint = rewritten
|
||||
model_used = app.state.llm_model
|
||||
else:
|
||||
final_hint = hint_static
|
||||
model_used = "none"
|
||||
|
||||
latency_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
return HintAskResponse(
|
||||
hint=hint,
|
||||
hint=final_hint,
|
||||
hint_static=hint_static,
|
||||
hint_rewritten=hint_rewritten,
|
||||
level=payload.level,
|
||||
puzzle_id=payload.puzzle_id,
|
||||
source=source,
|
||||
model_used=model_used,
|
||||
latency_ms=latency_ms,
|
||||
)
|
||||
except HTTPException as exc:
|
||||
status_code = exc.status_code
|
||||
@@ -288,6 +550,75 @@ def create_app(phrases_path: Optional[Path] = None) -> FastAPI:
|
||||
"level": payload.level,
|
||||
"session_id": payload.session_id,
|
||||
"client": request.client.host if request.client else None,
|
||||
"rewrite_requested": rewrite,
|
||||
"source": source,
|
||||
"model_used": model_used,
|
||||
"tokens_used": tokens_used,
|
||||
"safety_triggered": safety_triggered,
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000, 2),
|
||||
"status": status_code,
|
||||
})
|
||||
|
||||
@app.post("/hints/rewrite", response_model=HintRewriteResponse)
|
||||
async def hints_rewrite(
|
||||
payload: HintRewriteRequest, request: Request,
|
||||
) -> HintRewriteResponse:
|
||||
"""Debug-only path: forces the LLM rewrite and returns it.
|
||||
|
||||
Differs from `/hints/ask` in that it surfaces a 502 if the rewrite
|
||||
fails (no silent verbatim fallback). Useful for prompt tuning.
|
||||
"""
|
||||
started = time.perf_counter()
|
||||
status_code = 200
|
||||
source = "static"
|
||||
safety_triggered: Optional[str] = None
|
||||
tokens_used: Optional[int] = None
|
||||
try:
|
||||
hint_static = _resolve_static(payload.puzzle_id, payload.level)
|
||||
rewritten, source, safety_triggered, tokens_used = await _try_rewrite(
|
||||
hint_static,
|
||||
payload.puzzle_id,
|
||||
payload.level,
|
||||
max_tokens=payload.max_tokens or DEFAULT_LLM_MAX_TOKENS,
|
||||
temperature=(
|
||||
payload.temperature
|
||||
if payload.temperature is not None
|
||||
else DEFAULT_LLM_TEMPERATURE
|
||||
),
|
||||
)
|
||||
if rewritten is None:
|
||||
status_code = 502
|
||||
detail = (
|
||||
"rewrite blocked by safety filter"
|
||||
if safety_triggered
|
||||
else "rewrite failed (timeout/backend error)"
|
||||
)
|
||||
raise HTTPException(status_code=502, detail=detail)
|
||||
latency_ms = round((time.perf_counter() - started) * 1000, 2)
|
||||
return HintRewriteResponse(
|
||||
hint_static=hint_static,
|
||||
hint_rewritten=rewritten,
|
||||
level=payload.level,
|
||||
puzzle_id=payload.puzzle_id,
|
||||
model_used=app.state.llm_model,
|
||||
latency_ms=latency_ms,
|
||||
tokens_used=tokens_used,
|
||||
)
|
||||
except HTTPException as exc:
|
||||
status_code = exc.status_code
|
||||
raise
|
||||
finally:
|
||||
_emit_log({
|
||||
"ts": time.time(),
|
||||
"event": "hints_rewrite",
|
||||
"puzzle_id": payload.puzzle_id,
|
||||
"level": payload.level,
|
||||
"session_id": payload.session_id,
|
||||
"client": request.client.host if request.client else None,
|
||||
"source": source,
|
||||
"model_used": app.state.llm_model,
|
||||
"tokens_used": tokens_used,
|
||||
"safety_triggered": safety_triggered,
|
||||
"latency_ms": round((time.perf_counter() - started) * 1000, 2),
|
||||
"status": status_code,
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user