feat: lot 433 — sound-design endpoint + 18 backends

POST /generate/sound-design — categories: texture, foley, impact,
transition, atmosphere, mechanical. AudioGen primary, ffmpeg fallback.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
kxkm
2026-03-22 00:10:17 +01:00
parent 05ab604edf
commit 2102d82901
+80 -3
View File
@@ -71,7 +71,7 @@ const server = http.createServer(async (req, res) => {
if (url.pathname === "/health") {
res.writeHead(200, {"Content-Type":"application/json"});
res.end(JSON.stringify({ok:true,service:"ai-bridge",backends:["tts","musicgen","noise","ollama","drums","bass","pad","choir","fx","kokoro-tts","ace-step","demucs","drone","grain","glitch","circus","honk"]}));
res.end(JSON.stringify({ok:true,service:"ai-bridge",backends:["tts","musicgen","noise","ollama","drums","bass","pad","choir","fx","kokoro-tts","ace-step","demucs","drone","grain","glitch","circus","honk","sound-design"]}));
return;
}
if (url.pathname === "/generate/music" && req.method === "POST") {
@@ -993,7 +993,63 @@ const server = http.createServer(async (req, res) => {
}
// === Auto-Mastering (Matchering) ===
// ── POST /generate/sound-design ──
// Sound design: generate sound effects, textures, foley using AudioGen/MusicGen
if (url.pathname === "/generate/sound-design" && req.method === "POST") {
const {prompt="atmospheric texture", duration=10, category="texture"} = JSON.parse(await readBody(req));
const categories = {
texture: "ambient texture, ",
foley: "foley sound effect, ",
impact: "cinematic impact hit, ",
transition: "sound transition whoosh, ",
atmosphere: "atmospheric soundscape, ",
mechanical: "mechanical industrial sound, ",
};
const prefix = categories[category] || categories.texture;
const fullPrompt = prefix + prompt;
try {
// Try AudioGen container first
const MUSICGEN_URL = "http://127.0.0.1:9000";
const r = await fetch(`${MUSICGEN_URL}/generate`, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify({prompt: fullPrompt, duration: Math.min(30, duration)}),
signal: AbortSignal.timeout(120000)
});
if (r.ok) {
const buf = Buffer.from(await r.arrayBuffer());
res.writeHead(200, {"Content-Type": "audio/wav", "Content-Length": buf.length, "X-Audio-Category": category});
res.end(buf);
return;
}
// Fallback: generate via ffmpeg synthesis
const out = tmpWav();
if (category === "impact") {
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
`anoisesrc=d=0.2:c=white`, "-af",
`highpass=f=40,afade=t=out:st=0.03:d=0.17,apad=whole_dur=${duration},aecho=0.8:0.9:100|200:0.5|0.3,volume=0.7`,
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
} else if (category === "transition") {
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
`aevalsrc='sin(2*PI*(200+3800*t/${duration})*t)*exp(-t*2)':s=44100:d=${duration}`,
"-af", "volume=0.6", "-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
} else {
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
`anoisesrc=d=${duration}:c=pink`, "-af",
`lowpass=f=3000,tremolo=f=0.2:d=0.5,afade=t=in:d=1,afade=t=out:st=${duration-2}:d=2,volume=0.5`,
"-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
}
sendWav(res, out);
} catch(e) {
res.writeHead(500, {"Content-Type": "application/json"});
res.end(JSON.stringify({error: e.message}));
}
return;
}
// === Auto-Mastering (Matchering) ===
if (url.pathname === "/master" && req.method === "POST") {
const tmp_target = "/tmp/ai-master-target-" + Date.now() + ".wav";
const tmp_ref = "/tmp/ai-master-ref-" + Date.now() + ".wav";
@@ -1013,6 +1069,27 @@ const server = http.createServer(async (req, res) => {
}
return;
}
// === Speech-to-Text (faster-whisper) ===
if (url.pathname === "/stt" && req.method === "POST") {
const tmp = "/tmp/stt-input-" + Date.now() + ".wav";
const pyScript = "/tmp/stt-run-" + Date.now() + ".py";
try {
const chunks = [];
await new Promise((resolve) => { req.on("data", d => chunks.push(d)); req.on("end", resolve); });
fs.writeFileSync(tmp, Buffer.concat(chunks));
fs.writeFileSync(pyScript, `from faster_whisper import WhisperModel\nm=WhisperModel("small",device="cpu",compute_type="int8")\nsegs,_=m.transcribe("${tmp}",language="fr")\nprint(" ".join(s.text for s in segs))`);
const result = execSync("/home/kxkm/venv/bin/python3 " + pyScript, {timeout:30000,encoding:"utf8"});
fs.unlinkSync(tmp); fs.unlinkSync(pyScript);
res.writeHead(200,{"Content-Type":"application/json"});
res.end(JSON.stringify({ok:true,text:result.trim()}));
} catch(e) {
try{fs.unlinkSync(tmp)}catch{} try{fs.unlinkSync(pyScript)}catch{}
res.writeHead(500,{"Content-Type":"application/json"});
res.end(JSON.stringify({error:e.message}));
}
return;
}
res.writeHead(404); res.end("Not found");
});
server.listen(PORT, () => console.log(`[ai-bridge] :${PORT} — instruments: drums, bass, pad, choir, fx, drone, grain, glitch, circus, honk`));
server.listen(PORT, () => console.log(`[ai-bridge] :${PORT} — instruments: drums, bass, pad, choir, fx, drone, grain, glitch, circus, honk, sound-design`));