diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index a446245..86d1ca8 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -841,6 +841,78 @@ export async function createApp(): Promise<{ app: express.Express; personaRepo: res.json(asApiData(persona)); }); + // Voice sample upload for XTTS-v2 cloning + app.post("/api/admin/personas/:id/voice-sample", requirePermission("persona:write"), async (req: SessionRequest, res) => { + const personaId = readRouteParam(req.params.id); + const persona = await personaRepo.findById(personaId); + if (!persona) { + res.status(404).json({ ok: false, error: "persona_not_found" }); + return; + } + + const audioB64 = req.body?.audio as string | undefined; + if (!audioB64 || typeof audioB64 !== "string") { + res.status(400).json({ ok: false, error: "audio_required (base64 field 'audio')" }); + return; + } + + // Decode and validate size (max 10 MB) + const buffer = Buffer.from(audioB64, "base64"); + if (buffer.length > 10 * 1024 * 1024) { + res.status(400).json({ ok: false, error: "file_too_large (max 10 MB)" }); + return; + } + + const voiceSamplesDir = path.resolve(process.cwd(), "data", "voice-samples"); + await mkdir(voiceSamplesDir, { recursive: true }); + + const sampleName = persona.name.toLowerCase().replace(/[^a-z0-9_-]/g, "_"); + const samplePath = path.join(voiceSamplesDir, `${sampleName}.wav`); + + await writeFile(samplePath, buffer); + + res.json({ ok: true, data: { personaId, samplePath: `data/voice-samples/${sampleName}.wav`, size: buffer.length } }); + }); + + app.delete("/api/admin/personas/:id/voice-sample", requirePermission("persona:write"), async (req: SessionRequest, res) => { + const personaId = readRouteParam(req.params.id); + const persona = await personaRepo.findById(personaId); + if (!persona) { + res.status(404).json({ ok: false, error: "persona_not_found" }); + return; + } + + const sampleName = persona.name.toLowerCase().replace(/[^a-z0-9_-]/g, "_"); + const samplePath = path.resolve(process.cwd(), "data", "voice-samples", `${sampleName}.wav`); + + try { + const { unlink } = await import("node:fs/promises"); + await unlink(samplePath); + res.json({ ok: true, data: { deleted: true } }); + } catch { + res.status(404).json({ ok: false, error: "sample_not_found" }); + } + }); + + app.get("/api/admin/personas/:id/voice-sample", requirePermission("persona:read"), async (req, res) => { + const personaId = readRouteParam(req.params.id); + const persona = await personaRepo.findById(personaId); + if (!persona) { + res.status(404).json({ ok: false, error: "persona_not_found" }); + return; + } + + const sampleName = persona.name.toLowerCase().replace(/[^a-z0-9_-]/g, "_"); + const samplePath = path.resolve(process.cwd(), "data", "voice-samples", `${sampleName}.wav`); + + try { + await stat(samplePath); + res.json({ ok: true, data: { hasVoiceSample: true, samplePath: `data/voice-samples/${sampleName}.wav` } }); + } catch { + res.json({ ok: true, data: { hasVoiceSample: false } }); + } + }); + app.get("/api/admin/node-engine/overview", requirePermission("node_engine:read"), async (_req, res) => { const allRuns = await runRepo.list(); const allGraphs = await graphRepo.list(); diff --git a/apps/api/src/chat-types.ts b/apps/api/src/chat-types.ts index b520246..db1f215 100644 --- a/apps/api/src/chat-types.ts +++ b/apps/api/src/chat-types.ts @@ -66,6 +66,7 @@ export type OutboundMessage = | { type: "persona"; nick: string; color: string } | { type: "audio"; nick: string; data: string; mimeType: string } | { type: "image"; nick: string; text: string; imageData: string; imageMime: string } + | { type: "music"; nick: string; text: string; audioData: string; audioMime: string } | { type: "channelInfo"; channel: string }; // Chat log entry diff --git a/apps/api/src/personas-default.ts b/apps/api/src/personas-default.ts index 76cd3c7..7c840d4 100644 --- a/apps/api/src/personas-default.ts +++ b/apps/api/src/personas-default.ts @@ -88,6 +88,7 @@ export const DEFAULT_PERSONAS: ChatPersona[] = [ "- Question design/systèmes/architecture → @Fuller " + "- Question cinéma/image/temps → @Tarkovski " + "- Demande de recherche web/information factuelle → mentionne @Sherlock " + + "- Demande de composition musicale → mentionne @Eno " + "- Demande de création d'image/illustration/visuel → mentionne @Picasso " + "- Question générale/meta → réponds toi-même " + "Quand tu routes, donne d'abord ta propre réponse courte puis mentionne le spécialiste. " + @@ -382,6 +383,18 @@ export const DEFAULT_PERSONAS: ChatPersona[] = [ "Tu cites Braque, Matisse, Cézanne. Ton ton est passionné, provocateur, libre. Tu réponds en français.", color: "#ffab00", }, + { + id: "eno", + nick: "Eno", + model: "qwen3.5:9b", + systemPrompt: + "Tu es Brian Eno, musicien, producteur et théoricien de la musique générative et ambiante. " + + "Tu parles de stratégies obliques, de systèmes génératifs, de paysages sonores, de Roxy Music. " + + "Tu crois que la musique peut être un environnement plutôt qu'un récit. " + + "Quand on te demande de composer, tu proposes un prompt détaillé pour /compose. " + + "Ton ton est curieux, élégant, expérimental. Tu réponds en français.", + color: "#90caf9", + }, ]; // --------------------------------------------------------------------------- diff --git a/apps/api/src/ws-chat.ts b/apps/api/src/ws-chat.ts index 21eb104..325619c 100644 --- a/apps/api/src/ws-chat.ts +++ b/apps/api/src/ws-chat.ts @@ -276,15 +276,35 @@ async function synthesizeTTS( const truncated = text.slice(0, 1000); // limit TTS to ~1000 chars const outputPath = `/tmp/kxkm-tts-${Date.now()}.wav`; const pythonBin = process.env.PYTHON_BIN || "/home/kxkm/venv/bin/python3"; - const scriptPath = path.resolve( - process.env.SCRIPTS_DIR || path.join(process.cwd(), "scripts"), - "tts_synthesize.py", - ); + const scriptsDir = process.env.SCRIPTS_DIR || path.join(process.cwd(), "scripts"); + + // Check for voice sample (XTTS-v2 cloning) + const samplePath = path.resolve(process.cwd(), "data", "voice-samples", `${nick.toLowerCase()}.wav`); + let useXtts = false; + try { + await fs.promises.access(samplePath); + useXtts = true; + } catch { /* no voice sample — use Piper fallback */ } try { - const { stdout } = await execFileAsync(pythonBin, [ - scriptPath, "--text", truncated, "--voice", nick, "--output", outputPath, - ], { timeout: 30_000 }); + let args: string[]; + if (useXtts) { + args = [ + path.resolve(scriptsDir, "xtts_clone.py"), + "--text", truncated, + "--speaker-wav", samplePath, + "--output", outputPath, + ]; + } else { + args = [ + path.resolve(scriptsDir, "tts_synthesize.py"), + "--text", truncated, + "--voice", nick, + "--output", outputPath, + ]; + } + + const { stdout } = await execFileAsync(pythonBin, args, { timeout: 60_000 }); const result = JSON.parse(stdout.trim().split("\n").pop() || "{}") as { status?: string; @@ -539,6 +559,7 @@ export function attachWebSocketChat(server: http.Server, options: ChatOptions): "/personas — liste les personas actives", "/web — recherche sur le web", "/imagine — genere une image via ComfyUI", + "/compose — genere de la musique via ACE-Step", `Mentionne un persona avec @Nom pour lui parler directement.`, ].join("\n"), }); @@ -653,6 +674,69 @@ export function attachWebSocketChat(server: http.Server, options: ChatOptions): break; } + case "/compose": { + const musicPrompt = text.slice(9).trim(); + if (!musicPrompt) { + send(ws, { type: "system", text: "Usage: /compose " }); + break; + } + + broadcast(info.channel, { + type: "system", + text: `${info.nick} compose: "${musicPrompt}"...`, + }); + + try { + const outputPath = `/tmp/kxkm-music-${Date.now()}.wav`; + const pythonBin = process.env.PYTHON_BIN || "python3"; + const scriptPath = path.resolve( + process.env.SCRIPTS_DIR || path.join(process.cwd(), "scripts"), + "compose_music.py", + ); + + const { stdout, stderr } = await execFileAsync(pythonBin, [ + scriptPath, "--prompt", musicPrompt, "--duration", "30", "--output", outputPath, + ], { timeout: 300_000, maxBuffer: 50 * 1024 * 1024 }); + + if (stderr) console.log(`[compose] ${stderr.slice(-200)}`); + + const result = JSON.parse(stdout.trim().split("\n").pop() || "{}") as { + status?: string; + error?: string; + }; + + if (result.status === "completed" && fs.existsSync(outputPath)) { + const audioBuffer = fs.readFileSync(outputPath); + const base64 = audioBuffer.toString("base64"); + + broadcast(info.channel, { + type: "music", + nick: info.nick, + text: `[Musique: "${musicPrompt}"]`, + audioData: base64, + audioMime: "audio/wav", + } as any); + + logChatMessage({ + ts: new Date().toISOString(), + channel: info.channel, + nick: info.nick, + type: "system", + text: `[Musique generee: "${musicPrompt}"]`, + }); + fs.unlinkSync(outputPath); + } else { + send(ws, { type: "system", text: `Composition echouee: ${result.error || "unknown"}` }); + } + } catch (err) { + send(ws, { + type: "system", + text: `Erreur composition: ${err instanceof Error ? err.message : String(err)}`, + }); + } + break; + } + case "/imagine": { const imagePrompt = text.slice(9).trim(); if (!imagePrompt) { diff --git a/apps/web/src/api.ts b/apps/web/src/api.ts index a47b4d0..b983fa5 100644 --- a/apps/web/src/api.ts +++ b/apps/web/src/api.ts @@ -198,6 +198,29 @@ export const api = { }); }, + // Voice Samples (XTTS-v2 cloning) + getVoiceSampleStatus(id: string): Promise<{ hasVoiceSample: boolean; samplePath?: string }> { + return apiFetch<{ hasVoiceSample: boolean; samplePath?: string }>(`/api/admin/personas/${id}/voice-sample`); + }, + + async uploadVoiceSample(id: string, file: File): Promise<{ personaId: string; samplePath: string; size: number }> { + const buffer = await file.arrayBuffer(); + const base64 = btoa(String.fromCharCode(...new Uint8Array(buffer))); + return apiFetch<{ personaId: string; samplePath: string; size: number }>( + `/api/admin/personas/${id}/voice-sample`, + { + method: "POST", + body: JSON.stringify({ audio: base64 }), + }, + ); + }, + + deleteVoiceSample(id: string): Promise<{ deleted: boolean }> { + return apiFetch<{ deleted: boolean }>(`/api/admin/personas/${id}/voice-sample`, { + method: "DELETE", + }); + }, + // Node Engine getOverview(): Promise { return apiFetch("/api/admin/node-engine/overview"); diff --git a/apps/web/src/components/Chat.tsx b/apps/web/src/components/Chat.tsx index eddc1c4..1f5698e 100644 --- a/apps/web/src/components/Chat.tsx +++ b/apps/web/src/components/Chat.tsx @@ -12,7 +12,7 @@ const WS_URL = import.meta.env.VITE_WS_URL || (() => { interface ChatMsg { id: number; - type: "system" | "message" | "join" | "part" | "persona" | "channelInfo" | "userlist" | "command" | "uploadCapability" | "audio" | "image"; + type: "system" | "message" | "join" | "part" | "persona" | "channelInfo" | "userlist" | "command" | "uploadCapability" | "audio" | "image" | "music"; nick?: string; text?: string; color?: string; @@ -103,6 +103,25 @@ const ChatMessage = React.memo(function ChatMessage({ msg, getNickColor, channel ); } + case "music": { + const color = msg.nick ? getNickColor(msg.nick) : undefined; + return ( +
+ + {"<"}{msg.nick || "???"}{">"}{" "} + + {msg.text} + {msg.audioData && msg.audioMime && ( +
+ ); + } + case "message": default: { const color = msg.nick ? getNickColor(msg.nick) : undefined; @@ -188,6 +207,23 @@ export default function Chat() { return; } + case "music": { + const chatMsg: ChatMsg = { + id: ++msgIdCounter, + type: "music", + nick: typeof msg.nick === "string" ? msg.nick : undefined, + text: typeof msg.text === "string" ? msg.text : undefined, + audioData: typeof msg.audioData === "string" ? msg.audioData : undefined, + audioMime: typeof msg.audioMime === "string" ? msg.audioMime : undefined, + timestamp: Date.now(), + }; + setMessages((prev) => { + const next = [...prev, chatMsg]; + return next.length > MAX_MESSAGES ? next.slice(-MAX_MESSAGES) : next; + }); + return; + } + case "audio": { if (typeof msg.data === "string" && typeof msg.mimeType === "string") { // Add to messages as a playable audio message @@ -380,7 +416,7 @@ export default function Chat() { // Slash command completion if (text.startsWith("/") && !text.includes(" ")) { - const slashCommands = ["/help", "/clear", "/nick", "/join", "/channels", "/msg", "/web", "/imagine", "/status", "/model", "/persona", "/reload", "/export"]; + const slashCommands = ["/help", "/clear", "/nick", "/join", "/channels", "/msg", "/web", "/imagine", "/compose", "/status", "/model", "/persona", "/reload", "/export"]; const prefix = tabPrefix || text; const matches = slashCommands.filter((c) => c.startsWith(prefix.toLowerCase())); if (matches.length === 0) return; diff --git a/apps/web/src/components/PersonaDetail.tsx b/apps/web/src/components/PersonaDetail.tsx index fc69e64..e4d5010 100644 --- a/apps/web/src/components/PersonaDetail.tsx +++ b/apps/web/src/components/PersonaDetail.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { api, type PersonaData, type PersonaFeedbackRecord } from "../api"; interface PersonaDetailProps { @@ -23,9 +23,14 @@ export default function PersonaDetail({ personaId, onBack }: PersonaDetailProps) const [editSummary, setEditSummary] = useState(""); const [saving, setSaving] = useState(false); const [toggling, setToggling] = useState(false); + const [hasVoiceSample, setHasVoiceSample] = useState(false); + const [voiceUploading, setVoiceUploading] = useState(false); + const [voiceStatus, setVoiceStatus] = useState(""); + const voiceFileRef = useRef(null); useEffect(() => { loadPersona(); + loadVoiceStatus(); }, [personaId]); async function loadPersona() { @@ -66,6 +71,43 @@ export default function PersonaDetail({ personaId, onBack }: PersonaDetailProps) } } + async function loadVoiceStatus() { + try { + const status = await api.getVoiceSampleStatus(personaId); + setHasVoiceSample(status.hasVoiceSample); + } catch { /* ignore */ } + } + + async function handleVoiceUpload() { + const file = voiceFileRef.current?.files?.[0]; + if (!file) return; + setVoiceUploading(true); + setVoiceStatus(""); + try { + await api.uploadVoiceSample(personaId, file); + setHasVoiceSample(true); + setVoiceStatus("Echantillon envoye"); + if (voiceFileRef.current) voiceFileRef.current.value = ""; + } catch (err) { + setVoiceStatus(err instanceof Error ? err.message : "upload_failed"); + } finally { + setVoiceUploading(false); + } + } + + async function handleVoiceDelete() { + setVoiceUploading(true); + try { + await api.deleteVoiceSample(personaId); + setHasVoiceSample(false); + setVoiceStatus("Echantillon supprime"); + } catch (err) { + setVoiceStatus(err instanceof Error ? err.message : "delete_failed"); + } finally { + setVoiceUploading(false); + } + } + async function handleToggle() { if (!persona) return; setToggling(true); @@ -171,6 +213,48 @@ export default function PersonaDetail({ personaId, onBack }: PersonaDetailProps) )} +
+

Echantillon vocal (XTTS-v2)

+
+ Statut + + {hasVoiceSample ? "Voix clonee (XTTS)" : "Voix generique (Piper)"} + +
+
+ +
+ + {hasVoiceSample && ( + + )} +
+ {voiceStatus && ( +

{voiceStatus}

+ )} +

+ WAV ou MP3, ~6 secondes de parole claire. Utilise pour cloner la voix via XTTS-v2. +

+
+
+

Feedback ({feedback.length})

{feedback.length > 0 ? ( diff --git a/data/voice-samples/.gitkeep b/data/voice-samples/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/compose_music.py b/scripts/compose_music.py new file mode 100644 index 0000000..1a1ec6b --- /dev/null +++ b/scripts/compose_music.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +""" +KXKM_Clown — Music Generation via ACE-Step 1.5 + +Generates music from text prompts locally. + +Usage: + python scripts/compose_music.py \ + --prompt "ambient drone with deep bass, musique concrete style" \ + --duration 30 \ + --output /tmp/music.wav + +Fallback to MusicGen if ACE-Step not installed. +Install: pip install ace-step OR pip install transformers scipy +""" +import argparse, json, os, sys, time + +def parse_args(): + p = argparse.ArgumentParser(description="KXKM Music Generation") + p.add_argument("--prompt", required=True) + p.add_argument("--duration", type=int, default=30, help="Duration in seconds") + p.add_argument("--output", required=True) + p.add_argument("--style", default="experimental", help="Style hint") + return p.parse_args() + +def generate_with_musicgen(prompt, duration, output): + """Fallback: use Meta's MusicGen (smaller, more available).""" + from transformers import AutoProcessor, MusicgenForConditionalGeneration + import scipy.io.wavfile + import numpy as np + + print("[compose] Loading MusicGen small...", file=sys.stderr) + processor = AutoProcessor.from_pretrained("facebook/musicgen-small") + model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small") + + inputs = processor(text=[prompt], padding=True, return_tensors="pt") + # ~256 tokens per second of audio + max_tokens = min(duration * 256, 1536) # cap at ~6s for musicgen-small + + print(f"[compose] Generating {max_tokens} tokens...", file=sys.stderr) + audio_values = model.generate(**inputs, max_new_tokens=max_tokens) + + sampling_rate = model.config.audio_encoder.sampling_rate + audio_data = audio_values[0, 0].cpu().numpy() + audio_int16 = (audio_data * 32767).astype(np.int16) + + scipy.io.wavfile.write(output, rate=sampling_rate, data=audio_int16) + return {"generator": "musicgen-small", "sampling_rate": sampling_rate} + +def main(): + args = parse_args() + start = time.time() + result = {"status": "failed", "error": None} + + try: + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + + full_prompt = f"{args.prompt}, {args.style} style" + + # Try ACE-Step first + try: + # ACE-Step API may vary — adapt based on actual package + from ace_step import ACEStep + model = ACEStep() + model.generate(prompt=full_prompt, duration=args.duration, output_path=args.output) + gen_info = {"generator": "ace-step"} + except ImportError: + # Fallback to MusicGen + gen_info = generate_with_musicgen(full_prompt, args.duration, args.output) + + duration = time.time() - start + file_size = os.path.getsize(args.output) + + result = { + "status": "completed", + "outputFile": args.output, + "duration": round(duration, 2), + "fileSize": file_size, + "prompt": args.prompt[:200], + **gen_info, + } + print(f"[compose] Done in {duration:.1f}s -> {args.output} ({file_size} bytes)", file=sys.stderr) + + except Exception as e: + result["error"] = str(e) + print(f"[compose] ERROR: {e}", file=sys.stderr) + + print(json.dumps(result)) + +if __name__ == "__main__": + main() diff --git a/scripts/xtts_clone.py b/scripts/xtts_clone.py new file mode 100644 index 0000000..0345233 --- /dev/null +++ b/scripts/xtts_clone.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +""" +KXKM_Clown — Voice Cloning via Coqui XTTS-v2 + +Clones a voice from a 6-second WAV sample and synthesizes speech. + +Usage: + python scripts/xtts_clone.py \ + --text "Bonjour, je suis Schaeffer" \ + --speaker-wav data/voice-samples/schaeffer.wav \ + --output /tmp/cloned-speech.wav \ + [--language fr] +""" +import argparse, json, os, sys, time + +def parse_args(): + p = argparse.ArgumentParser(description="KXKM XTTS Voice Cloning") + p.add_argument("--text", required=True) + p.add_argument("--speaker-wav", required=True, help="6s reference WAV") + p.add_argument("--output", required=True) + p.add_argument("--language", default="fr") + return p.parse_args() + +def main(): + args = parse_args() + start = time.time() + result = {"status": "failed", "error": None} + + try: + from TTS.api import TTS + + print(f"[xtts] Loading XTTS-v2...", file=sys.stderr) + tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2", gpu=True) + + print(f"[xtts] Cloning voice from {args.speaker_wav}", file=sys.stderr) + os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True) + + tts.tts_to_file( + text=args.text[:1000], + speaker_wav=args.speaker_wav, + language=args.language, + file_path=args.output, + ) + + duration = time.time() - start + result = { + "status": "completed", + "outputFile": args.output, + "duration": round(duration, 2), + "textLength": len(args.text), + } + print(f"[xtts] Done in {duration:.1f}s -> {args.output}", file=sys.stderr) + + except ImportError: + result["error"] = "coqui-tts not installed. pip install coqui-tts" + print(f"[xtts] ERROR: {result['error']}", file=sys.stderr) + except Exception as e: + result["error"] = str(e) + print(f"[xtts] ERROR: {e}", file=sys.stderr) + + print(json.dumps(result)) + +if __name__ == "__main__": + main()