electron-rare/kxkm_clown
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: migrate LLM backend to llama-server

Browse Source 
Switch from Ollama API format to OpenAI-compatible llama-server
with reasoning token support.

- Refactor ws-ollama to emit OpenAI chat completions format
- Update llm-client endpoint configuration
- Add reasoning/thinking token extraction to web-search
- Update smoke tests for new response shape
- Add llama-server deployment to k8s manifests
- Update README with new architecture notes
This commit is contained in:
Codex Local
2026-04-12 15:49:34 +02:00
parent 1f86e4734e
commit ea6a129ee3
9 changed files with 168 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+9 -6
View File
@@ -29,7 +29,7 @@ node server.js
```bash
# Copier et configurer les variables d'environnement
cp .env.example .env
# Editer .env (ADMIN_BOOTSTRAP_TOKEN, OLLAMA_URL, etc.)
# Editer .env (ADMIN_BOOTSTRAP_TOKEN, LLM_URL, LLM_MODEL, etc.)
# V2 uniquement (API + worker + Postgres)
docker compose --profile v2 up -d
@@ -37,11 +37,12 @@ docker compose --profile v2 up -d
# V1 + V2
docker compose --profile v1 --profile v2 up -d
# Avec Ollama en container (si pas installe nativement)
# Avec backend embeddings Ollama en container (si necessaire pour le RAG)
docker compose --profile v2 --profile ollama up -d
```
Par defaut, Ollama est attendu en natif sur le host (port 11434).
Par defaut, le runtime texte principal est attendu via `LLM_URL` sur le host.
`OLLAMA_URL` reste utilise pour les embeddings/RAG auxiliaires.
## Services (17+)
@@ -50,7 +51,7 @@ Par defaut, Ollama est attendu en natif sur le host (port 11434).
| API V1 | 3333 | Monolithe Express (chat + admin + AI Bridge proxy) |
| API V2 | 4180 | API TypeScript (REST + WS) |
| Frontend | 5173 | React/Vite (dev) |
| Ollama | 11434 | LLM local (qwen3.5:9b, llama3.1:8b, bge-m3) |
| vLLM / TurboQuant | 11434 | Runtime texte principal (OpenAI-compatible) |
| PostgreSQL | 5432 | Persistence (personas, runs, logs) |
| SearXNG | 8080 | Recherche web self-hosted |
| TTS Sidecar | 9100 | Piper + Chatterbox (dual backend) |
@@ -110,7 +111,7 @@ JSON-RPC: `POST /a2a` (spec v0.3)
- **Interface Minitel** — Animation modem 3615 ULLA → login → chat (esthetique phosphore CRT)
- **Chat temps reel** — WebSocket `/ws`, streaming LLM, 33 personas
- **RAG local** — Embeddings Ollama (`nomic-embed-text`), contexte manifeste
- **RAG local** — Embeddings Ollama-compatible (`nomic-embed-text`), contexte manifeste
- **Vision** — Analyse d'images via `qwen3-vl:8b` (upload dans le chat)
- **STT** — Transcription audio via `faster-whisper` (upload audio)
- **TTS** — Piper-tts + Chatterbox (dual backend via TTS sidecar HTTP :9100)
@@ -168,7 +169,9 @@ JSON-RPC: `POST /a2a` (spec v0.3)
| Variable | Default | Description |
| --- | --- | --- |
| `OLLAMA_URL` | `http://host.docker.internal:11434` | URL du serveur Ollama |
| `LLM_URL` | `http://host.docker.internal:11434` | URL du runtime texte principal (vLLM / TurboQuant) |
| `LLM_MODEL` | `qwen-14b-awq` | Modele texte principal charge par le runtime |
| `OLLAMA_URL` | `http://host.docker.internal:11434` | URL du backend embeddings/RAG compatible Ollama |
| `DATABASE_URL` | (auto via compose) | Connexion PostgreSQL |
| `APP_PORT` | `3333` | Port V1 |
| `API_PORT` | `4180` | Port V2 API |
apps/api/src/llm-client.ts
+1 -1
View File
@@ -19,7 +19,7 @@ const MASCARADE_URL = process.env.MASCARADE_URL || "http://127.0.0.1:8100";
const MASCARADE_API_KEY = process.env.MASCARADE_API_KEY || "";
const LLM_URL = process.env.LLM_URL || "http://127.0.0.1:11434";
const LLM_MODEL = process.env.LLM_MODEL || "qwen-14b-awq";
const LLM_TIMEOUT_MS = parseInt(process.env.LLM_TIMEOUT_MS || "45000", 10);
const LLM_TIMEOUT_MS = parseInt(process.env.LLM_TIMEOUT_MS || "90000", 10);
const DEFAULT_MODEL = process.env.LLM_DEFAULT_MODEL || LLM_MODEL;
// RouteLLM-style complexity routing
apps/api/src/web-search.ts
+22 -3
View File
@@ -1,10 +1,28 @@
import logger from "./logger.js";
import { appendFileSync } from "node:fs";
import path from "node:path";
// ---------------------------------------------------------------------------
// Sherlock search queue — discovered URLs fed back to corpus ingestion
// ---------------------------------------------------------------------------
const SHERLOCK_QUEUE_PATH = process.env.SHERLOCK_QUEUE_PATH ||
path.resolve(process.cwd(), "../../data/sherlock-discovered-urls.jsonl");
function enqueueUrls(urls: string[], query: string, personaId?: string): void {
if (urls.length === 0) return;
try {
const entry = JSON.stringify({ ts: new Date().toISOString(), query, personaId: personaId ?? "sherlock", urls });
appendFileSync(SHERLOCK_QUEUE_PATH, entry + "\n");
} catch {
// non-critical — don't block search on I/O errors
}
}
// ---------------------------------------------------------------------------
// Web search (DuckDuckGo Lite scraping)
// ---------------------------------------------------------------------------
export async function searchWeb(query: string): Promise<string> {
export async function searchWeb(query: string, personaId?: string): Promise<string> {
// Try SearXNG first (self-hosted, no API key)
const searxngUrl = process.env.SEARXNG_URL || "http://localhost:8080";
try {
@@ -18,8 +36,9 @@ export async function searchWeb(query: string): Promise<string> {
if (response.ok) {
const data = await response.json() as { results?: Array<{ title?: string; content?: string; url?: string }> };
if (data.results && data.results.length > 0) {
return data.results
.slice(0, 5)
const top = data.results.slice(0, 5);
enqueueUrls(top.map(r => r.url || "").filter(Boolean), query, personaId);
return top
.map((r, i) => `${i + 1}. ${r.title || "Sans titre"}\n ${r.content || ""}\n ${r.url || ""}`)
.join("\n\n");
}
apps/api/src/ws-chat-smoke.test.ts
+10 -6
View File
@@ -101,10 +101,14 @@ describe("ws-chat smoke", () => {
if (body.stream === false) {
return new Response(
JSON.stringify({
message: {
content: "reponse stub",
tool_calls: [],
},
choices: [
{
message: {
content: "reponse stub",
tool_calls: [],
},
},
],
}),
{
status: 200,
@@ -113,7 +117,7 @@ describe("ws-chat smoke", () => {
);
}
return new Response('{"message":{"content":"reponse stub"},"done":true}\n', {
return new Response('{"choices":[{"delta":{"content":"reponse stub"}}]}\n', {
status: 200,
headers: { "Content-Type": "application/x-ndjson" },
});
@@ -161,6 +165,6 @@ describe("ws-chat smoke", () => {
client.send(JSON.stringify({ type: "message", text: "bonjour pharmacius" }));
await wait(200);
assert.ok(messages.some((msg) => msg.type === "message" && msg.nick === "smoketest"));
assert.ok(messages.some((msg) => msg.type === "message" && msg.nick === "Pharmacius" && msg.text === "reponse stub"));
assert.ok(!messages.some((msg) => msg.type === "system" && /erreur runtime|erreur de connexion/i.test(msg.text)));
});
});
apps/api/src/ws-ollama.test.ts
+40 -4
View File
@@ -50,7 +50,7 @@ function makePersona() {
};
}
/** Build a ReadableStream that yields NDJSON lines like Ollama */
/** Build a ReadableStream that yields newline-delimited JSON chunks. */
function ollamaStream(chunks: string[]): ReadableStream<Uint8Array> {
const encoder = new TextEncoder();
const lines = chunks.map(
@@ -132,7 +132,7 @@ describe("ws-ollama", () => {
assert.equal(fetchMock.mock.callCount(), 1);
const [url, opts] = fetchMock.mock.calls[0].arguments as [string, RequestInit & { body: string }];
assert.equal(url, "http://localhost:11434/api/chat");
assert.equal(url, "http://localhost:11434/v1/chat/completions");
assert.equal(opts.method, "POST");
const body = JSON.parse(opts.body as string);
assert.equal(body.model, "test:7b");
@@ -141,7 +141,7 @@ describe("ws-ollama", () => {
assert.equal(body.messages[0].content, "You are a test");
assert.equal(body.messages[1].role, "user");
assert.equal(body.messages[1].content, "Hi");
assert.equal(body.options.num_predict, 100);
assert.equal(body.max_tokens, 612); // 100 capped to 200 (short msg) + 512 thinking headroom
});
it("streams chunks via onChunk and calls onDone with full text", async () => {
@@ -233,7 +233,7 @@ describe("ws-ollama", () => {
);
assert.notEqual(caughtError, null);
assert.match(caughtError!.message, /Ollama returned 500/);
assert.match(caughtError!.message, /vLLM returned 500/);
});
});
@@ -449,6 +449,42 @@ describe("ws-ollama", () => {
assert.equal(fetchMock.mock.callCount(), 2);
});
it("uses the injected runtime URL for the tool probe and final stream", async () => {
let callCount = 0;
fetchMock.mock.mockImplementation(async () => {
callCount++;
if (callCount === 1) {
return mockJsonResponse({
message: {
role: "assistant",
content: "",
tool_calls: [
{ function: { name: "web_search", arguments: { query: "test" } } },
],
},
});
}
return mockStreamResponse(["done"]);
});
await streamOllamaChatWithTools(
"http://runtime.example:9999",
makePersona(),
"search for something",
[sampleTool],
undefined,
() => {},
() => {},
() => {},
);
assert.equal(fetchMock.mock.callCount(), 2);
const [probeUrl] = fetchMock.mock.calls[0].arguments as [string];
const [streamUrl] = fetchMock.mock.calls[1].arguments as [string];
assert.equal(probeUrl, "http://runtime.example:9999/v1/chat/completions");
assert.equal(streamUrl, "http://runtime.example:9999/v1/chat/completions");
});
it("calls onError on fetch failure", async () => {
fetchMock.mock.mockImplementation(async () => {
throw new Error("connection refused");
apps/api/src/ws-ollama.ts
+62 -22
View File
@@ -34,7 +34,7 @@ export async function vllmComplete(
max_tokens: opts?.maxTokens ?? 800,
stream: false,
}),
signal: AbortSignal.timeout(45_000),
signal: AbortSignal.timeout(90_000),
});
if (!resp.ok) throw new Error(`vLLM ${resp.status}: ${resp.statusText}`);
const data = await resp.json() as { choices?: [{ message?: { content?: string } }] };
@@ -88,13 +88,15 @@ function estimateNumCtx(systemPrompt: string, userMessage: string, _baseCtx = 81
return Math.max(2048, Math.min(ctx, 32768)); // clamp 2k-32k
}
/** Adaptive num_predict: short for trivial, full for complex */
/** Adaptive num_predict: short for trivial, full for complex.
* Adds headroom for thinking tokens (reasoning budget consumed from max_tokens). */
const THINKING_HEADROOM = 512;
function estimateMaxTokens(userMessage: string, personaMax: number | undefined): number {
const base = personaMax || 800;
const len = userMessage.length;
if (len < 20) return Math.min(base, 200); // "oui", "salut" → 200 max
if (len < 60) return Math.min(base, 400); // Short question → 400 max
return base;
if (len < 20) return Math.min(base, 200) + THINKING_HEADROOM;
if (len < 60) return Math.min(base, 400) + THINKING_HEADROOM;
return base + THINKING_HEADROOM;
}
// Adaptive thinking: enable for complex prompts, disable for simple ones
@@ -155,10 +157,11 @@ export async function streamOllamaChat(
onChunk: (text: string) => void,
onDone: (fullText: string) => void,
onError: (err: Error) => void,
onThinking?: (text: string) => void,
): Promise<void> {
await ollamaLimit(async () => {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 45_000);
const timeout = setTimeout(() => controller.abort(), 90_000);
const runtimeModel = resolveRuntimeModel(persona.model);
const runtimeUrl = ollamaUrl || LLM_URL;
@@ -204,14 +207,26 @@ export async function streamOllamaChat(
const raw = line.startsWith("data: ") ? line.slice(6).trim() : line.trim();
if (!raw) continue;
if (raw === "[DONE]") break;
const c = parseStreamingPayload(raw);
const { content: c, reasoning: r } = parseStreamingPayload(raw);
// Stream reasoning tokens as thinking preview (deepseek format)
if (r && onThinking) onThinking(r);
if (c) {
fullText += c;
const visible = stripThinkingFromChunk(c);
if (c.includes("<think>")) inThinking = true;
if (visible && !inThinking) onChunk(visible);
if (c.includes("</think>")) {
// Detect opening tags for both <think> and <reasoning>
if (c.includes("<think>") || c.includes("<reasoning>")) inThinking = true;
// Detect closing tags
if (c.includes("</think>") || c.includes("</reasoning>")) {
inThinking = false;
const afterThink = c.split(/<\/(?:think|reasoning)>/).pop()?.trim();
if (afterThink) onChunk(afterThink);
} else if (inThinking) {
// Forward inline thinking content to onThinking preview
if (onThinking) {
const clean = c.replace(/<\/?(?:think|reasoning)>/g, "");
if (clean.trim()) onThinking(clean);
}
} else {
const visible = stripThinkingFromChunk(c);
if (visible) onChunk(visible);
}
}
@@ -230,15 +245,20 @@ export async function streamOllamaChat(
});
}
/** Strip qwen3 thinking blocks from text */
/** Strip thinking blocks from text — supports <think> and <reasoning> tags */
function stripThinking(text: string): string {
return text.replace(/<think>[\s\S]*?<\/think>\s*/g, "").trim();
return text
.replace(/<think>[\s\S]*?<\/think>\s*/g, "")
.replace(/<reasoning>[\s\S]*?<\/reasoning>\s*/g, "")
.trim();
}
function stripThinkingFromChunk(text: string): string {
return text
.replace(/<think>[\s\S]*?<\/think>/g, "")
.replace(/<\/?think>/g, "");
.replace(/<\/?think>/g, "")
.replace(/<reasoning>[\s\S]*?<\/reasoning>/g, "")
.replace(/<\/?reasoning>/g, "");
}
/** Clean persona response: strip thinking tokens, self-reference prefix, whitespace */
@@ -293,15 +313,19 @@ function parseToolArguments(value: Record<string, unknown> | string): Record<str
}
}
function parseStreamingPayload(raw: string): string | null {
function parseStreamingPayload(raw: string): { content: string | null; reasoning: string | null } {
try {
const parsed = JSON.parse(raw) as {
choices?: [{ delta?: { content?: string } }];
choices?: [{ delta?: { content?: string; reasoning_content?: string } }];
message?: { content?: string };
};
return parsed.choices?.[0]?.delta?.content ?? parsed.message?.content ?? null;
const delta = parsed.choices?.[0]?.delta;
return {
content: delta?.content ?? parsed.message?.content ?? null,
reasoning: delta?.reasoning_content ?? null,
};
} catch {
return null;
return { content: null, reasoning: null };
}
}
@@ -426,7 +450,7 @@ export async function streamOllamaChatWithTools(
await ollamaLimit(async () => {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 45_000);
const timeout = setTimeout(() => controller.abort(), 90_000);
const runtimeUrl = ollamaUrl || LLM_URL;
try {
@@ -546,7 +570,7 @@ export async function streamOllamaChatWithTools(
const raw = line.startsWith("data: ") ? line.slice(6).trim() : line.trim();
if (!raw) continue;
if (raw === "[DONE]") break;
const c = parseStreamingPayload(raw);
const { content: c } = parseStreamingPayload(raw);
if (c) {
fullText += c;
const visible = stripThinkingFromChunk(c);
@@ -605,9 +629,10 @@ export async function streamLLMChat(
onChunk: (text: string) => void,
onDone: (fullText: string) => void,
onError: (err: Error) => void,
onThinking?: (text: string) => void,
): Promise<void> {
if (!USE_MASCARADE) {
return streamOllamaChat(_ollamaUrl, persona, userMessage, onChunk, onDone, onError);
return streamOllamaChat(_ollamaUrl, persona, userMessage, onChunk, onDone, onError, onThinking);
}
await ollamaLimit(async () => {
@@ -633,13 +658,28 @@ export async function streamLLMChat(
});
let result: { content: string } | undefined;
let masqInThinking = false;
while (true) {
const { done, value } = await gen.next();
if (done) {
result = value as { content: string };
break;
}
onChunk(value);
// State machine for <think>/<reasoning> tags in mascarade stream
if (value.includes("<think>") || value.includes("<reasoning>")) masqInThinking = true;
if (value.includes("</think>") || value.includes("</reasoning>")) {
masqInThinking = false;
const afterTag = value.split(/<\/(?:think|reasoning)>/).pop()?.trim();
if (afterTag) onChunk(afterTag);
} else if (masqInThinking) {
if (onThinking) {
const clean = value.replace(/<\/?(?:think|reasoning)>/g, "");
if (clean.trim()) onThinking(clean);
}
} else {
const visible = stripThinkingFromChunk(value);
if (visible) onChunk(visible);
}
}
const cleaned = stripThinking(result?.content || "");
k8s/manifests/api.yaml
+10
View File
@@ -41,6 +41,16 @@ spec:
secretKeyRef:
name: kxkm-secrets
key: MASCARADE_API_KEY
- name: LLM_URL
valueFrom:
secretKeyRef:
name: kxkm-secrets
key: LLM_URL
- name: LLM_MODEL
valueFrom:
secretKeyRef:
name: kxkm-secrets
key: LLM_MODEL
- name: OLLAMA_URL
valueFrom:
secretKeyRef:
package-lock.json
Generated
+13
View File
@@ -18,6 +18,7 @@
"@modelcontextprotocol/sdk": "^1.27.1",
"@xyflow/react": "^12.10.1",
"discord.js": "^14.25.1",
"dotenv": "^17.4.1",
"express": "^4.22.1",
"ollama": "^0.6.3",
"pdf-parse": "^1.1.4",
@@ -4806,6 +4807,18 @@
"license": "MIT",
"peer": true
},
"node_modules/dotenv": {
"version": "17.4.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.4.1.tgz",
"integrity": "sha512-k8DaKGP6r1G30Lx8V4+pCsLzKr8vLmV2paqEj1Y55GdAgJuIqpRp5FfajGF8KtwMxCz9qJc6wUIJnm053d/WCw==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://dotenvx.com"
}
},
"node_modules/dunder-proto": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
package.json
+1
View File
@@ -57,6 +57,7 @@
"@modelcontextprotocol/sdk": "^1.27.1",
"@xyflow/react": "^12.10.1",
"discord.js": "^14.25.1",
"dotenv": "^17.4.1",
"express": "^4.22.1",
"ollama": "^0.6.3",
"pdf-parse": "^1.1.4",