feat: lot 433 — sound-design endpoint + 18 backends
POST /generate/sound-design — categories: texture, foley, impact, transition, atmosphere, mechanical. AudioGen primary, ffmpeg fallback. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
+80
-3
@@ -71,7 +71,7 @@ const server = http.createServer(async (req, res) => {
|
||||
|
||||
if (url.pathname === "/health") {
|
||||
res.writeHead(200, {"Content-Type":"application/json"});
|
||||
res.end(JSON.stringify({ok:true,service:"ai-bridge",backends:["tts","musicgen","noise","ollama","drums","bass","pad","choir","fx","kokoro-tts","ace-step","demucs","drone","grain","glitch","circus","honk"]}));
|
||||
res.end(JSON.stringify({ok:true,service:"ai-bridge",backends:["tts","musicgen","noise","ollama","drums","bass","pad","choir","fx","kokoro-tts","ace-step","demucs","drone","grain","glitch","circus","honk","sound-design"]}));
|
||||
return;
|
||||
}
|
||||
if (url.pathname === "/generate/music" && req.method === "POST") {
|
||||
@@ -993,7 +993,63 @@ const server = http.createServer(async (req, res) => {
|
||||
}
|
||||
|
||||
|
||||
// === Auto-Mastering (Matchering) ===
|
||||
|
||||
// ── POST /generate/sound-design ──
|
||||
// Sound design: generate sound effects, textures, foley using AudioGen/MusicGen
|
||||
if (url.pathname === "/generate/sound-design" && req.method === "POST") {
|
||||
const {prompt="atmospheric texture", duration=10, category="texture"} = JSON.parse(await readBody(req));
|
||||
const categories = {
|
||||
texture: "ambient texture, ",
|
||||
foley: "foley sound effect, ",
|
||||
impact: "cinematic impact hit, ",
|
||||
transition: "sound transition whoosh, ",
|
||||
atmosphere: "atmospheric soundscape, ",
|
||||
mechanical: "mechanical industrial sound, ",
|
||||
};
|
||||
const prefix = categories[category] || categories.texture;
|
||||
const fullPrompt = prefix + prompt;
|
||||
try {
|
||||
// Try AudioGen container first
|
||||
const MUSICGEN_URL = "http://127.0.0.1:9000";
|
||||
const r = await fetch(`${MUSICGEN_URL}/generate`, {
|
||||
method: "POST",
|
||||
headers: {"Content-Type": "application/json"},
|
||||
body: JSON.stringify({prompt: fullPrompt, duration: Math.min(30, duration)}),
|
||||
signal: AbortSignal.timeout(120000)
|
||||
});
|
||||
if (r.ok) {
|
||||
const buf = Buffer.from(await r.arrayBuffer());
|
||||
res.writeHead(200, {"Content-Type": "audio/wav", "Content-Length": buf.length, "X-Audio-Category": category});
|
||||
res.end(buf);
|
||||
return;
|
||||
}
|
||||
// Fallback: generate via ffmpeg synthesis
|
||||
const out = tmpWav();
|
||||
if (category === "impact") {
|
||||
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
||||
`anoisesrc=d=0.2:c=white`, "-af",
|
||||
`highpass=f=40,afade=t=out:st=0.03:d=0.17,apad=whole_dur=${duration},aecho=0.8:0.9:100|200:0.5|0.3,volume=0.7`,
|
||||
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
|
||||
} else if (category === "transition") {
|
||||
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
||||
`aevalsrc='sin(2*PI*(200+3800*t/${duration})*t)*exp(-t*2)':s=44100:d=${duration}`,
|
||||
"-af", "volume=0.6", "-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
|
||||
} else {
|
||||
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
||||
`anoisesrc=d=${duration}:c=pink`, "-af",
|
||||
`lowpass=f=3000,tremolo=f=0.2:d=0.5,afade=t=in:d=1,afade=t=out:st=${duration-2}:d=2,volume=0.5`,
|
||||
"-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
|
||||
}
|
||||
sendWav(res, out);
|
||||
} catch(e) {
|
||||
res.writeHead(500, {"Content-Type": "application/json"});
|
||||
res.end(JSON.stringify({error: e.message}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
// === Auto-Mastering (Matchering) ===
|
||||
if (url.pathname === "/master" && req.method === "POST") {
|
||||
const tmp_target = "/tmp/ai-master-target-" + Date.now() + ".wav";
|
||||
const tmp_ref = "/tmp/ai-master-ref-" + Date.now() + ".wav";
|
||||
@@ -1013,6 +1069,27 @@ const server = http.createServer(async (req, res) => {
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// === Speech-to-Text (faster-whisper) ===
|
||||
if (url.pathname === "/stt" && req.method === "POST") {
|
||||
const tmp = "/tmp/stt-input-" + Date.now() + ".wav";
|
||||
const pyScript = "/tmp/stt-run-" + Date.now() + ".py";
|
||||
try {
|
||||
const chunks = [];
|
||||
await new Promise((resolve) => { req.on("data", d => chunks.push(d)); req.on("end", resolve); });
|
||||
fs.writeFileSync(tmp, Buffer.concat(chunks));
|
||||
fs.writeFileSync(pyScript, `from faster_whisper import WhisperModel\nm=WhisperModel("small",device="cpu",compute_type="int8")\nsegs,_=m.transcribe("${tmp}",language="fr")\nprint(" ".join(s.text for s in segs))`);
|
||||
const result = execSync("/home/kxkm/venv/bin/python3 " + pyScript, {timeout:30000,encoding:"utf8"});
|
||||
fs.unlinkSync(tmp); fs.unlinkSync(pyScript);
|
||||
res.writeHead(200,{"Content-Type":"application/json"});
|
||||
res.end(JSON.stringify({ok:true,text:result.trim()}));
|
||||
} catch(e) {
|
||||
try{fs.unlinkSync(tmp)}catch{} try{fs.unlinkSync(pyScript)}catch{}
|
||||
res.writeHead(500,{"Content-Type":"application/json"});
|
||||
res.end(JSON.stringify({error:e.message}));
|
||||
}
|
||||
return;
|
||||
}
|
||||
res.writeHead(404); res.end("Not found");
|
||||
});
|
||||
server.listen(PORT, () => console.log(`[ai-bridge] :${PORT} — instruments: drums, bass, pad, choir, fx, drone, grain, glitch, circus, honk`));
|
||||
server.listen(PORT, () => console.log(`[ai-bridge] :${PORT} — instruments: drums, bass, pad, choir, fx, drone, grain, glitch, circus, honk, sound-design`));
|
||||
|
||||
Reference in New Issue
Block a user