kxkm
5b62c9f12f
POST /generate/voice-clone — XTTS-v2 voice cloning (6s ref audio) 19 AI Bridge backends total Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1153 lines
52 KiB
JavaScript
1153 lines
52 KiB
JavaScript
const http = require("http");
|
|
const { execFileSync, execSync } = require("child_process");
|
|
const fs = require("fs");
|
|
const PORT = 8301;
|
|
const TTS_URL = "http://127.0.0.1:9100";
|
|
const OLLAMA_URL = "http://127.0.0.1:11434";
|
|
const KOKORO_URL = "http://127.0.0.1:9201";
|
|
const ACE_STEP_URL = "http://127.0.0.1:9200";
|
|
|
|
function readBody(req) {
|
|
return new Promise(r => { const c = []; req.on("data", d => c.push(d)); req.on("end", () => r(Buffer.concat(c).toString())); });
|
|
}
|
|
|
|
// ── Note-to-frequency conversion ──
|
|
const NOTE_MAP = { C:0, D:2, E:4, F:5, G:7, A:9, B:11 };
|
|
function noteToFreq(note) {
|
|
// e.g. "C2", "F#4", "Bb3"
|
|
const m = note.match(/^([A-G])(#|b)?(\d)$/i);
|
|
if (!m) return 220;
|
|
let semi = NOTE_MAP[m[1].toUpperCase()];
|
|
if (m[2] === '#') semi++;
|
|
if (m[2] === 'b') semi--;
|
|
const octave = parseInt(m[3]);
|
|
const midiNum = (octave + 1) * 12 + semi;
|
|
return 440 * Math.pow(2, (midiNum - 69) / 12);
|
|
}
|
|
|
|
// ── Chord note frequencies ──
|
|
function chordFreqs(chord) {
|
|
// "Cm" "Dm" "Am" "Em" or major like "C" "D"
|
|
const m = chord.match(/^([A-G])(#|b)?(m)?$/i);
|
|
if (!m) return [261.63, 311.13, 392.00]; // default Cm
|
|
const root = m[1].toUpperCase() + (m[2] || '') + '3';
|
|
const rootF = noteToFreq(root);
|
|
const isMinor = !!m[3];
|
|
// root, third, fifth, octave
|
|
const third = rootF * Math.pow(2, (isMinor ? 3 : 4) / 12);
|
|
const fifth = rootF * Math.pow(2, 7 / 12);
|
|
const oct = rootF * 2;
|
|
return [rootF, third, fifth, oct];
|
|
}
|
|
|
|
// ── Temp file helper ──
|
|
function tmpWav() { return `/tmp/ai-instrument-${Date.now()}-${Math.random().toString(36).slice(2,6)}.wav`; }
|
|
|
|
function cleanup(...files) { for (const f of files) try { fs.unlinkSync(f); } catch {} }
|
|
|
|
function sendWav(res, filePath) {
|
|
try {
|
|
const buf = fs.readFileSync(filePath);
|
|
cleanup(filePath);
|
|
res.writeHead(200, { "Content-Type": "audio/wav", "Content-Length": buf.length });
|
|
res.end(buf);
|
|
} catch (e) {
|
|
cleanup(filePath);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
}
|
|
|
|
const server = http.createServer(async (req, res) => {
|
|
res.setHeader("Access-Control-Allow-Origin", "*");
|
|
res.setHeader("Access-Control-Allow-Methods", "GET,POST,OPTIONS");
|
|
res.setHeader("Access-Control-Allow-Headers", "Content-Type");
|
|
if (req.method === "OPTIONS") { res.writeHead(200); res.end(); return; }
|
|
const url = new URL(req.url, `http://localhost:${PORT}`);
|
|
|
|
// ════════════════════════════════════════════════
|
|
// EXISTING ENDPOINTS (unchanged)
|
|
// ════════════════════════════════════════════════
|
|
|
|
if (url.pathname === "/health") {
|
|
res.writeHead(200, {"Content-Type":"application/json"});
|
|
res.end(JSON.stringify({ok:true,service:"ai-bridge",backends:["tts","musicgen","noise","ollama","drums","bass","pad","choir","fx","kokoro-tts","ace-step","demucs","drone","grain","glitch","circus","honk","sound-design","voice-clone"]}));
|
|
return;
|
|
}
|
|
if (url.pathname === "/generate/music" && req.method === "POST") {
|
|
const {prompt="ambient",duration=30,style="experimental"} = JSON.parse(await readBody(req));
|
|
// Try ACE-Step first (20s for 4min, MIT) → fallback to MusicGen
|
|
try {
|
|
const aceOk = await fetch(ACE_STEP_URL + '/health', {signal:AbortSignal.timeout(2000)}).then(r=>r.json()).catch(()=>null);
|
|
if (aceOk && aceOk.ok) {
|
|
const ar = await fetch(ACE_STEP_URL + '/generate', {method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({prompt:prompt+', '+style,duration,steps:8}),signal:AbortSignal.timeout(300000)});
|
|
if (ar.ok) { const buf = Buffer.from(await ar.arrayBuffer()); res.writeHead(200,{'Content-Type':'audio/wav','Content-Length':buf.length}); res.end(buf); return; }
|
|
}
|
|
} catch {} // ACE-Step unavailable, fall through
|
|
try {
|
|
const r = await fetch(`${TTS_URL}/compose`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({prompt:`${prompt}, ${style}`,duration}),signal:AbortSignal.timeout(300000)});
|
|
if (!r.ok) throw new Error(`TTS ${r.status}`);
|
|
const buf = Buffer.from(await r.arrayBuffer());
|
|
res.writeHead(200,{"Content-Type":"audio/wav","Content-Length":buf.length}); res.end(buf);
|
|
} catch(e) { res.writeHead(500,{"Content-Type":"application/json"}); res.end(JSON.stringify({error:e.message})); }
|
|
return;
|
|
}
|
|
if (url.pathname === "/generate/voice" && req.method === "POST") {
|
|
const {text="Bonjour",persona="pharmacius"} = JSON.parse(await readBody(req));
|
|
try {
|
|
const r = await fetch(`${TTS_URL}/synthesize`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({text,persona}),signal:AbortSignal.timeout(30000)});
|
|
if (!r.ok) throw new Error(`TTS ${r.status}`);
|
|
const buf = Buffer.from(await r.arrayBuffer());
|
|
res.writeHead(200,{"Content-Type":"audio/wav","Content-Length":buf.length}); res.end(buf);
|
|
} catch(e) { res.writeHead(500,{"Content-Type":"application/json"}); res.end(JSON.stringify({error:e.message})); }
|
|
return;
|
|
}
|
|
if (url.pathname === "/generate/noise" && req.method === "POST") {
|
|
const {type="pink",duration=10} = JSON.parse(await readBody(req));
|
|
const types = {white:"anoisesrc=d=DUR:c=white",pink:"anoisesrc=d=DUR:c=pink",brown:"anoisesrc=d=DUR:c=brown",sine:"sine=frequency=220:duration=DUR",drone:"sine=frequency=55:duration=DUR,tremolo=f=0.1:d=0.7"};
|
|
const filter = (types[type]||types.pink).replace(/DUR/g,String(duration));
|
|
const tmp = `/tmp/ai-noise-${Date.now()}.wav`;
|
|
try {
|
|
execFileSync("ffmpeg",["-f","lavfi","-i",filter,"-t",String(duration),"-ar","44100","-ac","2","-y",tmp],{timeout:15000});
|
|
const buf = fs.readFileSync(tmp); fs.unlinkSync(tmp);
|
|
res.writeHead(200,{"Content-Type":"audio/wav","Content-Length":buf.length}); res.end(buf);
|
|
} catch(e) { res.writeHead(500,{"Content-Type":"application/json"}); res.end(JSON.stringify({error:e.message})); }
|
|
return;
|
|
}
|
|
if (url.pathname === "/generate/suggest" && req.method === "POST") {
|
|
const {tracks=[]} = JSON.parse(await readBody(req));
|
|
const list = tracks.map((t,i)=>`#${i+1} [${t.type}] ${t.name} (${t.duration}s)`).join("\n");
|
|
try {
|
|
const r = await fetch(`${OLLAMA_URL}/api/chat`,{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({model:"qwen3.5:9b",messages:[{role:"user",content:`Compositeur. Pistes:\n${list}\nSuggere en JSON: {"type":"music|voice|noise","prompt":"...","duration":30}`}],stream:false,options:{num_predict:100},keep_alive:"30m",think:false}),signal:AbortSignal.timeout(15000)});
|
|
const d = await r.json();
|
|
res.writeHead(200,{"Content-Type":"application/json"}); res.end(d.message?.content||'{"type":"noise","prompt":"pink","duration":10}');
|
|
} catch { res.writeHead(200,{"Content-Type":"application/json"}); res.end('{"type":"noise","prompt":"pink","duration":10}'); }
|
|
return;
|
|
}
|
|
if (url.pathname === "/personas") {
|
|
try { const r = await fetch("http://localhost:3333/api/personas"); const d = await r.json(); res.writeHead(200,{"Content-Type":"application/json"}); res.end(JSON.stringify(d)); }
|
|
catch { res.writeHead(200,{"Content-Type":"application/json"}); res.end('{"data":[]}'); }
|
|
return;
|
|
}
|
|
|
|
// ════════════════════════════════════════════════
|
|
// NEW AI INSTRUMENT ENDPOINTS
|
|
// ════════════════════════════════════════════════
|
|
|
|
// ── 1. POST /instrument/drums ──
|
|
if (url.pathname === "/instrument/drums" && req.method === "POST") {
|
|
const { bpm = 120, pattern = "kick|snare|hihat", bars = 4, swing = 0 } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
const parts = [];
|
|
try {
|
|
const beatSec = 60 / bpm;
|
|
const totalBeats = bars * 4;
|
|
const totalDur = totalBeats * beatSec;
|
|
const instruments = pattern.split("|").map(s => s.trim().toLowerCase());
|
|
|
|
// Generate individual hit samples
|
|
const hitFiles = {};
|
|
if (instruments.includes("kick")) {
|
|
const f = tmpWav();
|
|
// Kick: sine 60Hz with quick pitch drop envelope, 50ms
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
"sine=frequency=60:duration=0.05,afade=t=out:st=0.01:d=0.04",
|
|
"-ar", "44100", "-ac", "2", "-y", f], { timeout: 5000 });
|
|
hitFiles.kick = f;
|
|
}
|
|
if (instruments.includes("snare")) {
|
|
const f = tmpWav();
|
|
// Snare: white noise 100ms with bandpass
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
"anoisesrc=d=0.1:c=white,bandpass=f=1200:w=800,afade=t=out:st=0.02:d=0.08",
|
|
"-ar", "44100", "-ac", "2", "-y", f], { timeout: 5000 });
|
|
hitFiles.snare = f;
|
|
}
|
|
if (instruments.includes("hihat")) {
|
|
const f = tmpWav();
|
|
// Hihat: high noise 30ms
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
"anoisesrc=d=0.03:c=white,highpass=f=8000,afade=t=out:st=0.005:d=0.025",
|
|
"-ar", "44100", "-ac", "2", "-y", f], { timeout: 5000 });
|
|
hitFiles.hihat = f;
|
|
}
|
|
|
|
// Build a silent base track
|
|
const baseTmp = tmpWav();
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anullsrc=r=44100:cl=stereo`, "-t", String(totalDur), "-y", baseTmp], { timeout: 5000 });
|
|
parts.push(baseTmp);
|
|
|
|
// Place hits on grid — build complex filter with adelay
|
|
// Pattern: kick on beats 1,3; snare on 2,4; hihat on every 8th
|
|
let inputs = ["-i", baseTmp];
|
|
let filterParts = [];
|
|
let mixInputs = ["[0]"];
|
|
let inputIdx = 1;
|
|
|
|
for (let beat = 0; beat < totalBeats; beat++) {
|
|
const swingOffset = (beat % 2 === 1) ? (swing * beatSec * 0.33) : 0;
|
|
const timeMs = Math.round((beat * beatSec + swingOffset) * 1000);
|
|
|
|
// Kick on 1 and 3 of each bar
|
|
if (instruments.includes("kick") && (beat % 4 === 0 || beat % 4 === 2)) {
|
|
inputs.push("-i", hitFiles.kick);
|
|
filterParts.push(`[${inputIdx}]adelay=${timeMs}|${timeMs}[dk${beat}]`);
|
|
mixInputs.push(`[dk${beat}]`);
|
|
inputIdx++;
|
|
}
|
|
// Snare on 2 and 4
|
|
if (instruments.includes("snare") && (beat % 4 === 1 || beat % 4 === 3)) {
|
|
inputs.push("-i", hitFiles.snare);
|
|
filterParts.push(`[${inputIdx}]adelay=${timeMs}|${timeMs}[sn${beat}]`);
|
|
mixInputs.push(`[sn${beat}]`);
|
|
inputIdx++;
|
|
}
|
|
// Hihat on every 8th note (every half beat)
|
|
if (instruments.includes("hihat") && beat % 1 === 0) {
|
|
inputs.push("-i", hitFiles.hihat);
|
|
filterParts.push(`[${inputIdx}]adelay=${timeMs}|${timeMs}[hh${beat}]`);
|
|
mixInputs.push(`[hh${beat}]`);
|
|
inputIdx++;
|
|
// Also add off-beat hihat
|
|
const offMs = Math.round((beat * beatSec + beatSec * 0.5) * 1000);
|
|
if (offMs < totalDur * 1000) {
|
|
inputs.push("-i", hitFiles.hihat);
|
|
filterParts.push(`[${inputIdx}]adelay=${offMs}|${offMs}[hho${beat}]`);
|
|
mixInputs.push(`[hho${beat}]`);
|
|
inputIdx++;
|
|
}
|
|
}
|
|
}
|
|
|
|
const filterGraph = filterParts.join(";") +
|
|
";" + mixInputs.join("") + `amix=inputs=${mixInputs.length}:duration=first:normalize=0`;
|
|
|
|
execFileSync("ffmpeg", [...inputs, "-filter_complex", filterGraph,
|
|
"-t", String(totalDur), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
|
|
cleanup(...Object.values(hitFiles), baseTmp);
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out, ...Object.values(parts));
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// ── 2. POST /instrument/bass ──
|
|
if (url.pathname === "/instrument/bass" && req.method === "POST") {
|
|
const { note = "C2", waveform = "sine", duration = 10, pattern = "sustain", bpm = 120 } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
try {
|
|
const freq = noteToFreq(note);
|
|
const beatSec = 60 / bpm;
|
|
|
|
let filterSrc;
|
|
const wave = waveform.toLowerCase();
|
|
if (wave === "saw") {
|
|
// Sawtooth approximation: sum of harmonics
|
|
const harmonics = Array.from({length:6}, (_, i) => {
|
|
const h = i + 1;
|
|
return `sine=frequency=${freq*h}:duration=${duration}[h${h}];[h${h}]volume=${1/h}[vh${h}]`;
|
|
});
|
|
filterSrc = harmonics.join(";") + ";" +
|
|
Array.from({length:6}, (_, i) => `[vh${i+1}]`).join("") +
|
|
`amix=inputs=6:normalize=0`;
|
|
} else if (wave === "square") {
|
|
// Square approximation: odd harmonics
|
|
const harmonics = Array.from({length:4}, (_, i) => {
|
|
const h = 2*i + 1;
|
|
return `sine=frequency=${freq*h}:duration=${duration}[h${h}];[h${h}]volume=${1/h}[vh${h}]`;
|
|
});
|
|
filterSrc = harmonics.join(";") + ";" +
|
|
Array.from({length:4}, (_, i) => `[vh${2*i+1}]`).join("") +
|
|
`amix=inputs=4:normalize=0`;
|
|
} else {
|
|
filterSrc = `sine=frequency=${freq}:duration=${duration}`;
|
|
}
|
|
|
|
let filterPost = "";
|
|
if (pattern === "pulse") {
|
|
// Rhythmic tremolo at BPM rate
|
|
const tremoloFreq = bpm / 60;
|
|
filterPost = `,tremolo=f=${tremoloFreq}:d=0.8`;
|
|
} else if (pattern === "arp") {
|
|
// Ascending arpeggio: root, minor third, fifth, octave — each 1 beat
|
|
const notes = [freq, freq * Math.pow(2, 3/12), freq * Math.pow(2, 7/12), freq * 2];
|
|
const stepDur = beatSec;
|
|
const segments = [];
|
|
const segFiles = [];
|
|
for (let i = 0; i < Math.ceil(duration / (stepDur * notes.length)); i++) {
|
|
for (const nf of notes) {
|
|
const sf = tmpWav();
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`sine=frequency=${nf}:duration=${stepDur}`,
|
|
"-af", "afade=t=in:d=0.01,afade=t=out:st=" + (stepDur - 0.02) + ":d=0.02",
|
|
"-ar", "44100", "-ac", "2", "-y", sf], { timeout: 5000 });
|
|
segFiles.push(sf);
|
|
}
|
|
}
|
|
// Concat all segments
|
|
const concatInput = segFiles.map(f => `-i|${f}`).join("|").split("|");
|
|
const concatFilter = segFiles.map((_, i) => `[${i}]`).join("") +
|
|
`concat=n=${segFiles.length}:v=0:a=1`;
|
|
execFileSync("ffmpeg", [...concatInput, "-filter_complex", concatFilter,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
cleanup(...segFiles);
|
|
sendWav(res, out);
|
|
return;
|
|
}
|
|
|
|
// For sine/saw/square with sustain or pulse
|
|
if (wave === "saw" || wave === "square") {
|
|
// Complex filter graph — write to intermediate then apply post
|
|
const inter = tmpWav();
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i", filterSrc.split(";")[0].split(",")[0].includes("sine=") ? `sine=frequency=${freq}:duration=${duration}` : `sine=frequency=${freq}:duration=${duration}`,
|
|
"-f", "lavfi", "-i", `sine=frequency=${freq}:duration=${duration}`,
|
|
"-filter_complex", filterSrc + filterPost,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
} else {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
filterSrc + filterPost + `,afade=t=in:d=0.02,afade=t=out:st=${duration-0.1}:d=0.1`,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
}
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// ── 3. POST /instrument/pad ──
|
|
if (url.pathname === "/instrument/pad" && req.method === "POST") {
|
|
const { type = "warm", duration = 30, chord = "Cm" } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
try {
|
|
const freqs = chordFreqs(chord);
|
|
// Detune each note slightly for thickness
|
|
const detuneCents = [-8, -3, 3, 8];
|
|
|
|
// Type determines tonal shaping
|
|
const typeParams = {
|
|
warm: { lowpass: 800, tremRate: 0.15, tremDepth: 0.3 },
|
|
cold: { lowpass: 3000, tremRate: 0.08, tremDepth: 0.2 },
|
|
dark: { lowpass: 400, tremRate: 0.1, tremDepth: 0.4 },
|
|
bright: { lowpass: 8000, tremRate: 0.2, tremDepth: 0.2 },
|
|
evolving:{ lowpass: 2000, tremRate: 0.05, tremDepth: 0.6 },
|
|
};
|
|
const tp = typeParams[type] || typeParams.warm;
|
|
|
|
// Generate each detuned voice
|
|
const voiceFiles = [];
|
|
for (let i = 0; i < freqs.length; i++) {
|
|
const f = freqs[i] * Math.pow(2, detuneCents[i % detuneCents.length] / 1200);
|
|
const vf = tmpWav();
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`sine=frequency=${f.toFixed(4)}:duration=${duration}`,
|
|
"-af", `lowpass=f=${tp.lowpass},tremolo=f=${tp.tremRate}:d=${tp.tremDepth},afade=t=in:d=2,afade=t=out:st=${duration-3}:d=3,volume=0.4`,
|
|
"-ar", "44100", "-ac", "2", "-y", vf], { timeout: 10000 });
|
|
voiceFiles.push(vf);
|
|
}
|
|
|
|
// Mix all voices together
|
|
const mixInputs = voiceFiles.flatMap(f => ["-i", f]);
|
|
const mixFilter = voiceFiles.map((_, i) => `[${i}]`).join("") +
|
|
`amix=inputs=${voiceFiles.length}:normalize=0,aecho=0.8:0.7:60:0.4`;
|
|
|
|
execFileSync("ffmpeg", [...mixInputs, "-filter_complex", mixFilter,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
|
|
cleanup(...voiceFiles);
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// ── 4. POST /instrument/choir ──
|
|
if (url.pathname === "/instrument/choir" && req.method === "POST") {
|
|
const { text = "aaah", voices = 3, spread = 0.5, duration = 10 } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
const voiceFiles = [];
|
|
try {
|
|
// Generate base voice via TTS
|
|
const baseTmp = tmpWav();
|
|
let hasTTS = false;
|
|
try {
|
|
const r = await fetch(`${TTS_URL}/synthesize`, {
|
|
method: "POST",
|
|
headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ text, persona: "pharmacius" }),
|
|
signal: AbortSignal.timeout(10000)
|
|
});
|
|
if (r.ok) {
|
|
fs.writeFileSync(baseTmp, Buffer.from(await r.arrayBuffer()));
|
|
hasTTS = true;
|
|
}
|
|
} catch {}
|
|
|
|
if (!hasTTS) {
|
|
// Fallback: generate a vocal-like sine drone
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`sine=frequency=220:duration=${duration},tremolo=f=5:d=0.3`,
|
|
"-af", `afade=t=in:d=0.5,afade=t=out:st=${duration-1}:d=1`,
|
|
"-ar", "44100", "-ac", "2", "-y", baseTmp], { timeout: 10000 });
|
|
}
|
|
|
|
// Create multiple voices with pitch shifts and delays
|
|
for (let v = 0; v < voices; v++) {
|
|
const vf = tmpWav();
|
|
// Spread pitch: center voice + spread up/down
|
|
const pitchShift = (v - Math.floor(voices / 2)) * spread * 100; // cents
|
|
const delayMs = Math.round(v * 30 * spread); // slight delay for spread
|
|
const asetrate = 44100 * Math.pow(2, pitchShift / 1200);
|
|
|
|
execFileSync("ffmpeg", ["-i", baseTmp, "-af",
|
|
`asetrate=${asetrate.toFixed(0)},aresample=44100,atempo=1,adelay=${delayMs}|${delayMs},volume=${0.7 / voices}`,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", vf], { timeout: 10000 });
|
|
voiceFiles.push(vf);
|
|
}
|
|
|
|
// Mix all choir voices
|
|
const mixInputs = voiceFiles.flatMap(f => ["-i", f]);
|
|
const mixFilter = voiceFiles.map((_, i) => `[${i}]`).join("") +
|
|
`amix=inputs=${voiceFiles.length}:normalize=0,aecho=0.8:0.6:80:0.5`;
|
|
|
|
execFileSync("ffmpeg", [...mixInputs, "-filter_complex", mixFilter,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
|
|
cleanup(baseTmp, ...voiceFiles);
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out, ...voiceFiles);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// ── 5. POST /instrument/fx ──
|
|
if (url.pathname === "/instrument/fx" && req.method === "POST") {
|
|
const { type = "riser", duration = 5 } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
try {
|
|
let filter;
|
|
switch (type) {
|
|
case "riser":
|
|
// Ascending sine sweep from 100Hz to 4000Hz
|
|
filter = `sine=frequency=100:duration=${duration}[s];[s]afreqshift=shift=${3900/duration}*t`;
|
|
// Use asetrate trick for sweep
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`aevalsrc='sin(2*PI*(100+3900*t/${duration})*t)':s=44100:d=${duration}`,
|
|
"-af", `afade=t=in:d=0.1,afade=t=out:st=${duration-0.3}:d=0.3,volume=0.8`,
|
|
"-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
break;
|
|
|
|
case "drop":
|
|
// Descending freq sweep with distortion
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`aevalsrc='sin(2*PI*(4000-3800*t/${duration})*t)':s=44100:d=${duration}`,
|
|
"-af", `afade=t=in:d=0.02,afade=t=out:st=${duration-0.5}:d=0.5,overdrive=gain=8:colour=50,lowpass=f=2000,volume=0.6`,
|
|
"-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
break;
|
|
|
|
case "sweep":
|
|
// Bandpass filter sweep on noise
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=${duration}:c=white`,
|
|
"-af", `afftfilt=real='hypot(re,im)*cos(2*PI*clip((200+3800*t/${duration}),200,4000)/sr*n)':imag='hypot(re,im)*sin(2*PI*clip((200+3800*t/${duration}),200,4000)/sr*n)',afade=t=in:d=0.5,afade=t=out:st=${duration-1}:d=1,volume=0.5`,
|
|
"-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
break;
|
|
|
|
case "impact":
|
|
// Short burst + reverb tail
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=0.15:c=white`,
|
|
"-af", `highpass=f=40,afade=t=out:st=0.02:d=0.13,apad=whole_dur=${duration},aecho=0.8:0.9:100|200|300:0.6|0.4|0.2,volume=0.7`,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
break;
|
|
|
|
case "stutter":
|
|
// Rapid repeated short noise bursts
|
|
{
|
|
const burstDur = 0.03;
|
|
const gapDur = 0.07;
|
|
const cycleDur = burstDur + gapDur;
|
|
const cycles = Math.floor(duration / cycleDur);
|
|
// Generate single burst
|
|
const burst = tmpWav();
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=${burstDur}:c=white`,
|
|
"-af", `afade=t=out:st=0.01:d=0.02,apad=whole_dur=${cycleDur}`,
|
|
"-ar", "44100", "-ac", "2", "-y", burst], { timeout: 5000 });
|
|
// Loop it
|
|
execFileSync("ffmpeg", ["-stream_loop", String(cycles - 1), "-i", burst,
|
|
"-t", String(duration), "-af", `volume=0.7,afade=t=in:d=0.2,afade=t=out:st=${duration-0.5}:d=0.5`,
|
|
"-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
cleanup(burst);
|
|
}
|
|
break;
|
|
|
|
default:
|
|
// Fallback to riser
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`aevalsrc='sin(2*PI*(100+3900*t/${duration})*t)':s=44100:d=${duration}`,
|
|
"-af", "volume=0.8",
|
|
"-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
}
|
|
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
|
|
// === Kokoro fast TTS ===
|
|
if (url.pathname === "/generate/voice-fast" && req.method === "POST") {
|
|
const {text="Hello",voice="af_heart",speed=1.0} = JSON.parse(await readBody(req));
|
|
try {
|
|
const r = await fetch(KOKORO_URL + "/synthesize", {
|
|
method:"POST", headers:{"Content-Type":"application/json"},
|
|
body:JSON.stringify({text,voice,speed}), signal:AbortSignal.timeout(15000)
|
|
});
|
|
if (!r.ok) throw new Error("Kokoro " + r.status);
|
|
const buf = Buffer.from(await r.arrayBuffer());
|
|
res.writeHead(200,{"Content-Type":"audio/wav","Content-Length":buf.length}); res.end(buf);
|
|
} catch(e) { res.writeHead(500,{"Content-Type":"application/json"}); res.end(JSON.stringify({error:e.message})); }
|
|
return;
|
|
}
|
|
|
|
// === Kokoro voice list ===
|
|
if (url.pathname === "/voices" && req.method === "GET") {
|
|
try {
|
|
const r = await fetch(KOKORO_URL + "/voices");
|
|
const d = await r.json();
|
|
res.writeHead(200,{"Content-Type":"application/json"}); res.end(JSON.stringify(d));
|
|
} catch { res.writeHead(200,{"Content-Type":"application/json"}); res.end('{"voices":["af_heart","am_michael"]}'); }
|
|
return;
|
|
}
|
|
|
|
// === ACE-Step music gen ===
|
|
if (url.pathname === "/generate/music-ai" && req.method === "POST") {
|
|
const body = JSON.parse(await readBody(req));
|
|
try {
|
|
const r = await fetch(ACE_STEP_URL + "/generate", {
|
|
method:"POST", headers:{"Content-Type":"application/json"},
|
|
body:JSON.stringify(body), signal:AbortSignal.timeout(300000)
|
|
});
|
|
if (!r.ok) throw new Error("ACE-Step " + r.status);
|
|
const buf = Buffer.from(await r.arrayBuffer());
|
|
res.writeHead(200,{"Content-Type":"audio/wav","Content-Length":buf.length,"X-Elapsed-Ms":r.headers.get("X-Elapsed-Ms")||"0"}); res.end(buf);
|
|
} catch(e) { res.writeHead(500,{"Content-Type":"application/json"}); res.end(JSON.stringify({error:e.message})); }
|
|
return;
|
|
}
|
|
|
|
// === Demucs stem separation ===
|
|
if (url.pathname === "/separate" && req.method === "POST") {
|
|
const tmp = "/tmp/demucs-input-" + Date.now() + ".wav";
|
|
const outDir = "/tmp/demucs-out-" + Date.now();
|
|
try {
|
|
// Read binary audio body
|
|
const chunks = [];
|
|
await new Promise((resolve) => { req.on("data", d => chunks.push(d)); req.on("end", resolve); });
|
|
const inputBuf = Buffer.concat(chunks);
|
|
fs.writeFileSync(tmp, inputBuf);
|
|
// Run demucs
|
|
execSync("/home/kxkm/ai/ace-step/venv/bin/python -m demucs -n htdemucs --two-stems vocals -o " + outDir + " " + tmp, {timeout:120000});
|
|
// Find output stems
|
|
const stemDir = fs.readdirSync(outDir + "/htdemucs").find(d => d.startsWith("demucs-input"));
|
|
const stems = {};
|
|
if (stemDir) {
|
|
for (const f of fs.readdirSync(outDir + "/htdemucs/" + stemDir)) {
|
|
const stemBuf = fs.readFileSync(outDir + "/htdemucs/" + stemDir + "/" + f);
|
|
stems[f.replace(".wav","")] = stemBuf.toString("base64");
|
|
}
|
|
}
|
|
// Cleanup
|
|
fs.unlinkSync(tmp);
|
|
fs.rmSync(outDir, {recursive:true,force:true});
|
|
res.writeHead(200,{"Content-Type":"application/json"});
|
|
res.end(JSON.stringify({ok:true,stems:Object.keys(stems),data:stems}));
|
|
} catch(e) {
|
|
try { fs.unlinkSync(tmp); } catch{}
|
|
try { fs.rmSync(outDir,{recursive:true,force:true}); } catch{}
|
|
res.writeHead(500,{"Content-Type":"application/json"}); res.end(JSON.stringify({error:e.message}));
|
|
}
|
|
return;
|
|
}
|
|
|
|
|
|
|
|
// ════════════════════════════════════════════════
|
|
// KXKM CLOWN INSTRUMENTS
|
|
// ════════════════════════════════════════════════
|
|
|
|
// ── POST /instrument/drone ──
|
|
// Thick evolving drone with unison, LFO modulation, long attack/release
|
|
if (url.pathname === "/instrument/drone" && req.method === "POST") {
|
|
const { note = "C2", duration = 30, voices = 5, detune = 8, lfoRate = 0.07, lfoDepth = 0.4, cutoff = 1200, waveform = "saw" } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
const voiceFiles = [];
|
|
try {
|
|
const baseFreq = noteToFreq(note);
|
|
const detuneSpread = voices > 1 ? detune : 0;
|
|
for (let v = 0; v < voices; v++) {
|
|
const vf = tmpWav();
|
|
const spread = voices > 1 ? (v / (voices - 1)) * 2 - 1 : 0;
|
|
const cents = spread * detuneSpread;
|
|
const freq = baseFreq * Math.pow(2, cents / 1200);
|
|
const pan = spread * 0.8;
|
|
const panL = Math.cos((pan + 1) * Math.PI * 0.25);
|
|
const panR = Math.sin((pan + 1) * Math.PI * 0.25);
|
|
let src;
|
|
if (waveform === "saw") {
|
|
const harmonics = Array.from({length: 8}, (_, i) => {
|
|
const h = i + 1;
|
|
return `sin(2*PI*${(freq * h).toFixed(4)}*t)/${h}`;
|
|
}).join("+");
|
|
src = `aevalsrc='(${harmonics})*0.3':s=44100:d=${duration}`;
|
|
} else if (waveform === "square") {
|
|
const harmonics = Array.from({length: 6}, (_, i) => {
|
|
const h = 2 * i + 1;
|
|
return `sin(2*PI*${(freq * h).toFixed(4)}*t)/${h}`;
|
|
}).join("+");
|
|
src = `aevalsrc='(${harmonics})*0.4':s=44100:d=${duration}`;
|
|
} else {
|
|
src = `sine=frequency=${freq.toFixed(4)}:duration=${duration}`;
|
|
}
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i", src,
|
|
"-af", [
|
|
`lowpass=f=${cutoff}`,
|
|
`tremolo=f=${Math.max(0.1, lfoRate)}:d=${lfoDepth}`,
|
|
`afade=t=in:d=${Math.min(duration * 0.3, 8)}`,
|
|
`afade=t=out:st=${duration - Math.min(duration * 0.3, 8)}:d=${Math.min(duration * 0.3, 8)}`,
|
|
`volume=${(0.6 / voices).toFixed(3)}`,
|
|
`pan=stereo|c0=${panL.toFixed(3)}*c0|c1=${panR.toFixed(3)}*c0`
|
|
].join(","),
|
|
"-ar", "44100", "-ac", "2", "-y", vf], { timeout: 15000 });
|
|
voiceFiles.push(vf);
|
|
}
|
|
const mixInputs = voiceFiles.flatMap(f => ["-i", f]);
|
|
const mixFilter = voiceFiles.map((_, i) => `[${i}]`).join("") +
|
|
`amix=inputs=${voiceFiles.length}:normalize=0,aecho=0.8:0.7:80:0.3`;
|
|
execFileSync("ffmpeg", [...mixInputs, "-filter_complex", mixFilter,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 20000 });
|
|
cleanup(...voiceFiles);
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out, ...voiceFiles);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// ── POST /instrument/grain ──
|
|
// Granular texture: chops source audio into micro-grains, scatters them
|
|
if (url.pathname === "/instrument/grain" && req.method === "POST") {
|
|
const { source = "noise", duration = 15, grainSize = 0.05, density = 20, pitchSpread = 400, stereo = 0.8 } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
const grainFiles = [];
|
|
try {
|
|
// Generate source material
|
|
const srcFile = tmpWav();
|
|
const srcDur = Math.max(2, duration * 0.3);
|
|
if (source === "tone") {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`sine=frequency=220:duration=${srcDur}`,
|
|
"-af", "tremolo=f=3:d=0.5",
|
|
"-ar", "44100", "-ac", "2", "-y", srcFile], { timeout: 10000 });
|
|
} else if (source === "voice") {
|
|
try {
|
|
const r = await fetch(`${TTS_URL}/synthesize`, {
|
|
method: "POST", headers: { "Content-Type": "application/json" },
|
|
body: JSON.stringify({ text: "aaaaah oooooh eeeeeh", persona: "pharmacius" }),
|
|
signal: AbortSignal.timeout(15000)
|
|
});
|
|
if (r.ok) fs.writeFileSync(srcFile, Buffer.from(await r.arrayBuffer()));
|
|
else throw new Error("TTS fail");
|
|
} catch {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=${srcDur}:c=pink`,
|
|
"-ar", "44100", "-ac", "2", "-y", srcFile], { timeout: 10000 });
|
|
}
|
|
} else {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=${srcDur}:c=pink`,
|
|
"-af", "lowpass=f=4000",
|
|
"-ar", "44100", "-ac", "2", "-y", srcFile], { timeout: 10000 });
|
|
}
|
|
|
|
// Generate grains: extract tiny slices at random positions with random pitch
|
|
const totalGrains = Math.min(Math.floor(duration * density), 500);
|
|
for (let g = 0; g < totalGrains; g++) {
|
|
const gf = tmpWav();
|
|
const startPos = Math.random() * Math.max(0.01, srcDur - grainSize);
|
|
const pitchCents = (Math.random() - 0.5) * 2 * pitchSpread;
|
|
const asetrate = Math.round(44100 * Math.pow(2, pitchCents / 1200));
|
|
const pan = (Math.random() - 0.5) * 2 * stereo;
|
|
const panL = Math.cos((pan + 1) * Math.PI * 0.25).toFixed(3);
|
|
const panR = Math.sin((pan + 1) * Math.PI * 0.25).toFixed(3);
|
|
const timeMs = Math.round((g / totalGrains) * duration * 1000);
|
|
try {
|
|
execFileSync("ffmpeg", ["-ss", startPos.toFixed(3), "-i", srcFile,
|
|
"-t", String(grainSize),
|
|
"-af", [
|
|
`asetrate=${asetrate},aresample=44100`,
|
|
`afade=t=in:d=${grainSize * 0.3}`,
|
|
`afade=t=out:st=${grainSize * 0.7}:d=${grainSize * 0.3}`,
|
|
`volume=0.5`,
|
|
`pan=stereo|c0=${panL}*c0|c1=${panR}*c0`,
|
|
`apad=whole_dur=${duration}`,
|
|
`adelay=${timeMs}|${timeMs}`
|
|
].join(","),
|
|
"-ar", "44100", "-ac", "2", "-y", gf], { timeout: 5000 });
|
|
grainFiles.push(gf);
|
|
} catch { cleanup(gf); }
|
|
}
|
|
|
|
if (grainFiles.length === 0) throw new Error("No grains generated");
|
|
|
|
// Mix in batches of 30 to avoid ffmpeg filter limit
|
|
let currentFiles = [...grainFiles];
|
|
while (currentFiles.length > 1) {
|
|
const batchFiles = [];
|
|
for (let b = 0; b < currentFiles.length; b += 30) {
|
|
const batch = currentFiles.slice(b, b + 30);
|
|
if (batch.length === 1) { batchFiles.push(batch[0]); continue; }
|
|
const bf = tmpWav();
|
|
const batchInputs = batch.flatMap(f => ["-i", f]);
|
|
const batchFilter = batch.map((_, i) => `[${i}]`).join("") +
|
|
`amix=inputs=${batch.length}:normalize=0`;
|
|
execFileSync("ffmpeg", [...batchInputs, "-filter_complex", batchFilter,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", bf], { timeout: 30000 });
|
|
batchFiles.push(bf);
|
|
if (b > 0) batch.forEach(f => cleanup(f));
|
|
}
|
|
currentFiles = batchFiles;
|
|
}
|
|
|
|
// Final fade
|
|
const finalSrc = currentFiles[0];
|
|
execFileSync("ffmpeg", ["-i", finalSrc,
|
|
"-af", `afade=t=in:d=0.5,afade=t=out:st=${duration - 1}:d=1,volume=1.5`,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
|
|
cleanup(srcFile, ...grainFiles, ...currentFiles.filter(f => f !== out));
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out, ...grainFiles);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// ── POST /instrument/glitch ──
|
|
// Buffer repeat, stutter, bit crush, downsample effects
|
|
if (url.pathname === "/instrument/glitch" && req.method === "POST") {
|
|
const { duration = 10, bpm = 140, crushBits = 8, stutter = 0.6, reverse = 0.3, source = "noise" } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
const segFiles = [];
|
|
try {
|
|
// Generate source material
|
|
const srcFile = tmpWav();
|
|
if (source === "tone") {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`aevalsrc='sin(2*PI*110*t)+sin(2*PI*165*t)*0.5+sin(2*PI*220*t)*0.3':s=44100:d=${duration}`,
|
|
"-ar", "44100", "-ac", "2", "-y", srcFile], { timeout: 10000 });
|
|
} else {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=${duration}:c=pink`,
|
|
"-af", "lowpass=f=6000,volume=3,alimiter=limit=0.9",
|
|
"-ar", "44100", "-ac", "2", "-y", srcFile], { timeout: 10000 });
|
|
}
|
|
|
|
// Slice into beat-sized chunks and process each differently
|
|
const beatSec = 60 / bpm;
|
|
const sliceDur = beatSec * 0.5; // 8th notes
|
|
const numSlices = Math.floor(duration / sliceDur);
|
|
|
|
for (let s = 0; s < numSlices; s++) {
|
|
const sf = tmpWav();
|
|
const startTime = (s * sliceDur) % Math.max(sliceDur, duration - sliceDur);
|
|
const doStutter = Math.random() < stutter;
|
|
const doReverse = Math.random() < reverse;
|
|
const crushAmount = Math.pow(2, Math.floor(crushBits + Math.random() * (16 - crushBits)));
|
|
|
|
let filters = [];
|
|
if (doReverse) filters.push("areverse");
|
|
// Bit crush simulation
|
|
filters.push(`acrusher=bits=${Math.max(2, Math.floor(crushBits + Math.random() * 4))}:mode=log:aa=1`);
|
|
// Random pitch shift
|
|
const pitchRatio = 1 + (Math.random() - 0.5) * 0.5;
|
|
filters.push(`asetrate=${Math.round(44100 * pitchRatio)},aresample=44100`);
|
|
filters.push(`volume=0.6`);
|
|
|
|
if (doStutter) {
|
|
// Repeat: take first 1/4 of the slice and loop it 4 times
|
|
const stutterDur = sliceDur * 0.25;
|
|
try {
|
|
const stutterSrc = tmpWav();
|
|
execFileSync("ffmpeg", ["-ss", startTime.toFixed(3), "-i", srcFile,
|
|
"-t", String(stutterDur), "-af", filters.join(","),
|
|
"-ar", "44100", "-ac", "2", "-y", stutterSrc], { timeout: 5000 });
|
|
execFileSync("ffmpeg", ["-stream_loop", "3", "-i", stutterSrc,
|
|
"-t", String(sliceDur),
|
|
"-ar", "44100", "-ac", "2", "-y", sf], { timeout: 5000 });
|
|
cleanup(stutterSrc);
|
|
} catch {
|
|
execFileSync("ffmpeg", ["-ss", startTime.toFixed(3), "-i", srcFile,
|
|
"-t", String(sliceDur), "-af", filters.join(","),
|
|
"-ar", "44100", "-ac", "2", "-y", sf], { timeout: 5000 });
|
|
}
|
|
} else {
|
|
execFileSync("ffmpeg", ["-ss", startTime.toFixed(3), "-i", srcFile,
|
|
"-t", String(sliceDur), "-af", filters.join(","),
|
|
"-ar", "44100", "-ac", "2", "-y", sf], { timeout: 5000 });
|
|
}
|
|
segFiles.push(sf);
|
|
}
|
|
|
|
// Concat all segments
|
|
const listFile = `/tmp/ai-glitch-list-${Date.now()}.txt`;
|
|
fs.writeFileSync(listFile, segFiles.map(f => `file '${f}'`).join("\n"));
|
|
execFileSync("ffmpeg", ["-f", "concat", "-safe", "0", "-i", listFile,
|
|
"-af", `afade=t=in:d=0.1,afade=t=out:st=${duration - 0.3}:d=0.3`,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 20000 });
|
|
|
|
cleanup(srcFile, listFile, ...segFiles);
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out, ...segFiles);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// ── POST /instrument/circus ──
|
|
// Barrel organ / calliope: organ pipe harmonics with tremolo and air noise
|
|
if (url.pathname === "/instrument/circus" && req.method === "POST") {
|
|
const { notes = "C4,E4,G4", duration = 15, register = "principal", wobble = 0.15, air = 0.2, tremoloRate = 5, tremoloDepth = 0.3 } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
const pipeFiles = [];
|
|
try {
|
|
const noteList = notes.split(",").map(n => n.trim());
|
|
// Register defines harmonic content
|
|
const registers = {
|
|
"flute8": [1, 0.2, 0.05],
|
|
"flute4": [1, 0.3, 0.1, 0.05],
|
|
"principal": [1, 0.7, 0.5, 0.35, 0.2, 0.1],
|
|
"mixture": [1, 0.8, 0.6, 0.5, 0.4, 0.3, 0.2, 0.15, 0.1, 0.05]
|
|
};
|
|
const harmonicAmps = registers[register] || registers.principal;
|
|
|
|
for (const noteName of noteList) {
|
|
const freq = noteToFreq(noteName);
|
|
const pf = tmpWav();
|
|
// Build additive synthesis with harmonics
|
|
const harmonicExpr = harmonicAmps.map((amp, i) => {
|
|
const h = i + 1;
|
|
const detuneHz = (Math.random() - 0.5) * wobble * freq * 0.01;
|
|
return `sin(2*PI*${(freq * h + detuneHz).toFixed(4)}*t)*${amp}`;
|
|
}).join("+");
|
|
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`aevalsrc='(${harmonicExpr})*0.3':s=44100:d=${duration}`,
|
|
"-af", [
|
|
`tremolo=f=${tremoloRate}:d=${tremoloDepth}`,
|
|
`afade=t=in:d=0.05`,
|
|
`afade=t=out:st=${duration - 0.2}:d=0.2`,
|
|
`volume=${(0.7 / noteList.length).toFixed(3)}`
|
|
].join(","),
|
|
"-ar", "44100", "-ac", "2", "-y", pf], { timeout: 15000 });
|
|
pipeFiles.push(pf);
|
|
}
|
|
|
|
// Add air/breath noise
|
|
const airFile = tmpWav();
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=${duration}:c=pink`,
|
|
"-af", `highpass=f=2000,lowpass=f=8000,volume=${air},afade=t=in:d=0.1,afade=t=out:st=${duration-0.3}:d=0.3`,
|
|
"-ar", "44100", "-ac", "2", "-y", airFile], { timeout: 10000 });
|
|
pipeFiles.push(airFile);
|
|
|
|
// Mix
|
|
const mixInputs = pipeFiles.flatMap(f => ["-i", f]);
|
|
const mixFilter = pipeFiles.map((_, i) => `[${i}]`).join("") +
|
|
`amix=inputs=${pipeFiles.length}:normalize=0`;
|
|
execFileSync("ffmpeg", [...mixInputs, "-filter_complex", mixFilter,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 20000 });
|
|
|
|
cleanup(...pipeFiles);
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out, ...pipeFiles);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// ── POST /instrument/honk ──
|
|
// Klaxon / siren / horn with frequency sweep
|
|
if (url.pathname === "/instrument/honk" && req.method === "POST") {
|
|
const { mode = "klaxon", duration = 5, frequency = 440, sweepRange = 1200, sweepTime = 0.5, richness = 0.6, noise = 0.15 } = JSON.parse(await readBody(req));
|
|
const out = tmpWav();
|
|
try {
|
|
const freq = frequency;
|
|
const sweepCents = sweepRange;
|
|
const sweepSec = sweepTime;
|
|
let toneExpr;
|
|
switch (mode) {
|
|
case "siren": {
|
|
// Continuous sine sweep up and down
|
|
const sweepRatio = Math.pow(2, sweepCents / 1200);
|
|
const halfFreq = freq / Math.sqrt(sweepRatio);
|
|
const topFreq = freq * Math.sqrt(sweepRatio);
|
|
toneExpr = `aevalsrc='(sin(2*PI*${halfFreq}*pow(${(topFreq/halfFreq).toFixed(6)},mod(t/${sweepSec},2)/2 + (mod(t/${sweepSec},2)>1)*(1-mod(t/${sweepSec},2)/2+0.5))*t)`;
|
|
// Simpler: just use a triangle LFO on frequency
|
|
toneExpr = `aevalsrc='sin(2*PI*(${freq}+${freq*(sweepRatio-1)*0.5}*sin(2*PI*t/${sweepSec*2}))*t)`;
|
|
// Add harmonics
|
|
if (richness > 0.1) {
|
|
toneExpr += `+sin(2*PI*2*(${freq}+${freq*(sweepRatio-1)*0.5}*sin(2*PI*t/${sweepSec*2}))*t)*${richness*0.5}`;
|
|
toneExpr += `+sin(2*PI*3*(${freq}+${freq*(sweepRatio-1)*0.5}*sin(2*PI*t/${sweepSec*2}))*t)*${richness*0.25}`;
|
|
}
|
|
toneExpr += `':s=44100:d=${duration}`;
|
|
break;
|
|
}
|
|
case "horn": {
|
|
// Quick attack, steady pitch, slight vibrato
|
|
const vibratoHz = 4;
|
|
const vibratoCents = 15;
|
|
const vibratoRatio = Math.pow(2, vibratoCents / 1200) - 1;
|
|
toneExpr = `aevalsrc='sin(2*PI*${freq}*(1+${vibratoRatio}*sin(2*PI*${vibratoHz}*t))*t)`;
|
|
if (richness > 0.1) {
|
|
toneExpr += `+sin(2*PI*2*${freq}*(1+${vibratoRatio}*sin(2*PI*${vibratoHz}*t))*t)*${richness*0.5}`;
|
|
toneExpr += `+sin(2*PI*3*${freq}*t)*${richness*0.3}`;
|
|
toneExpr += `+sin(2*PI*4*${freq}*t)*${richness*0.15}`;
|
|
}
|
|
toneExpr += `':s=44100:d=${duration}`;
|
|
break;
|
|
}
|
|
default: {
|
|
// Klaxon: alternating two tones using square-ish modulation via abs(sin)
|
|
const highFreq = freq * Math.pow(2, sweepCents / 1200);
|
|
const switchRate = 1 / sweepSec;
|
|
const midFreq = (freq + highFreq) / 2;
|
|
const halfRange = (highFreq - freq) / 2;
|
|
// Use sign approximation: sgn(sin(x)) ≈ sin(x)/max(abs(sin(x)),0.01) clamped
|
|
// Simpler: use floor(mod()) to get 0/1 square wave
|
|
toneExpr = `aevalsrc='sin(2*PI*(${midFreq}+${halfRange}*(2*floor(mod(t*${switchRate},1)+0.5)-1))*t)`;
|
|
if (richness > 0.1) {
|
|
toneExpr += `+sin(2*PI*2*(${midFreq}+${halfRange}*(2*floor(mod(t*${switchRate},1)+0.5)-1))*t)*${richness*0.4}`;
|
|
toneExpr += `+sin(2*PI*3*(${midFreq}+${halfRange}*(2*floor(mod(t*${switchRate},1)+0.5)-1))*t)*${richness*0.2}`;
|
|
}
|
|
toneExpr += `':s=44100:d=${duration}`;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Generate tone
|
|
const toneFile = tmpWav();
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i", toneExpr,
|
|
"-af", `afade=t=in:d=0.01,afade=t=out:st=${duration-0.15}:d=0.15,volume=0.6`,
|
|
"-ar", "44100", "-ac", "2", "-y", toneFile], { timeout: 15000 });
|
|
|
|
if (noise > 0.01) {
|
|
// Add noise component
|
|
const noiseFile = tmpWav();
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=${duration}:c=white`,
|
|
"-af", `bandpass=f=${freq}:w=${freq*0.5},volume=${noise},afade=t=in:d=0.01,afade=t=out:st=${duration-0.15}:d=0.15`,
|
|
"-ar", "44100", "-ac", "2", "-y", noiseFile], { timeout: 10000 });
|
|
|
|
// Mix tone + noise
|
|
execFileSync("ffmpeg", ["-i", toneFile, "-i", noiseFile,
|
|
"-filter_complex", "[0][1]amix=inputs=2:normalize=0",
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], { timeout: 15000 });
|
|
cleanup(toneFile, noiseFile);
|
|
} else {
|
|
fs.renameSync(toneFile, out);
|
|
}
|
|
|
|
sendWav(res, out);
|
|
} catch (e) {
|
|
cleanup(out);
|
|
res.writeHead(500, { "Content-Type": "application/json" });
|
|
res.end(JSON.stringify({ error: e.message }));
|
|
}
|
|
return;
|
|
}
|
|
|
|
|
|
|
|
// ── POST /generate/sound-design ──
|
|
// Sound design: generate sound effects, textures, foley using AudioGen/MusicGen
|
|
if (url.pathname === "/generate/sound-design" && req.method === "POST") {
|
|
const {prompt="atmospheric texture", duration=10, category="texture"} = JSON.parse(await readBody(req));
|
|
const categories = {
|
|
texture: "ambient texture, ",
|
|
foley: "foley sound effect, ",
|
|
impact: "cinematic impact hit, ",
|
|
transition: "sound transition whoosh, ",
|
|
atmosphere: "atmospheric soundscape, ",
|
|
mechanical: "mechanical industrial sound, ",
|
|
};
|
|
const prefix = categories[category] || categories.texture;
|
|
const fullPrompt = prefix + prompt;
|
|
try {
|
|
// Try AudioGen container first
|
|
const MUSICGEN_URL = "http://127.0.0.1:9000";
|
|
const r = await fetch(`${MUSICGEN_URL}/generate`, {
|
|
method: "POST",
|
|
headers: {"Content-Type": "application/json"},
|
|
body: JSON.stringify({prompt: fullPrompt, duration: Math.min(30, duration)}),
|
|
signal: AbortSignal.timeout(120000)
|
|
});
|
|
if (r.ok) {
|
|
const buf = Buffer.from(await r.arrayBuffer());
|
|
res.writeHead(200, {"Content-Type": "audio/wav", "Content-Length": buf.length, "X-Audio-Category": category});
|
|
res.end(buf);
|
|
return;
|
|
}
|
|
// Fallback: generate via ffmpeg synthesis
|
|
const out = tmpWav();
|
|
if (category === "impact") {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=0.2:c=white`, "-af",
|
|
`highpass=f=40,afade=t=out:st=0.03:d=0.17,apad=whole_dur=${duration},aecho=0.8:0.9:100|200:0.5|0.3,volume=0.7`,
|
|
"-t", String(duration), "-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
|
|
} else if (category === "transition") {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`aevalsrc='sin(2*PI*(200+3800*t/${duration})*t)*exp(-t*2)':s=44100:d=${duration}`,
|
|
"-af", "volume=0.6", "-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
|
|
} else {
|
|
execFileSync("ffmpeg", ["-f", "lavfi", "-i",
|
|
`anoisesrc=d=${duration}:c=pink`, "-af",
|
|
`lowpass=f=3000,tremolo=f=0.2:d=0.5,afade=t=in:d=1,afade=t=out:st=${duration-2}:d=2,volume=0.5`,
|
|
"-ar", "44100", "-ac", "2", "-y", out], {timeout: 15000});
|
|
}
|
|
sendWav(res, out);
|
|
} catch(e) {
|
|
res.writeHead(500, {"Content-Type": "application/json"});
|
|
res.end(JSON.stringify({error: e.message}));
|
|
}
|
|
return;
|
|
}
|
|
|
|
|
|
|
|
// ── POST /generate/voice-clone ──
|
|
// XTTS-v2 voice cloning: 6s reference audio → clone voice
|
|
if (url.pathname === "/generate/voice-clone" && req.method === "POST") {
|
|
const body = JSON.parse(await readBody(req));
|
|
const { text = "Bonjour", reference_audio, language = "fr" } = body;
|
|
if (!text) { res.writeHead(400, {"Content-Type":"application/json"}); res.end(JSON.stringify({error:"text required"})); return; }
|
|
|
|
const out = tmpWav();
|
|
try {
|
|
// Use Coqui TTS XTTS-v2 via Python
|
|
const pyScript = `
|
|
import sys, json
|
|
from TTS.api import TTS
|
|
tts = TTS("tts_models/multilingual/multi-dataset/xtts_v2", gpu=True)
|
|
ref = sys.argv[1] if len(sys.argv) > 1 and sys.argv[1] != "none" else None
|
|
text = sys.argv[2]
|
|
lang = sys.argv[3]
|
|
out = sys.argv[4]
|
|
if ref:
|
|
tts.tts_to_file(text=text, file_path=out, speaker_wav=ref, language=lang)
|
|
else:
|
|
tts.tts_to_file(text=text, file_path=out, language=lang)
|
|
print(json.dumps({"ok": True}))
|
|
`;
|
|
const pyFile = `/tmp/xtts-gen-${Date.now()}.py`;
|
|
fs.writeFileSync(pyFile, pyScript);
|
|
|
|
let refFile = "none";
|
|
if (reference_audio) {
|
|
refFile = `/tmp/xtts-ref-${Date.now()}.wav`;
|
|
fs.writeFileSync(refFile, Buffer.from(reference_audio, "base64"));
|
|
}
|
|
|
|
execSync(`/home/kxkm/venv/bin/python ${pyFile} "${refFile}" "${text.replace(/"/g, '\\"').slice(0, 500)}" "${language}" "${out}"`, {
|
|
timeout: 120000,
|
|
env: { ...process.env, CUDA_VISIBLE_DEVICES: "0" }
|
|
});
|
|
|
|
cleanup(pyFile);
|
|
if (refFile !== "none") cleanup(refFile);
|
|
|
|
if (fs.existsSync(out)) {
|
|
sendWav(res, out);
|
|
} else {
|
|
res.writeHead(500, {"Content-Type":"application/json"});
|
|
res.end(JSON.stringify({error: "XTTS generation failed — no output"}));
|
|
}
|
|
} catch(e) {
|
|
cleanup(out);
|
|
res.writeHead(500, {"Content-Type":"application/json"});
|
|
res.end(JSON.stringify({error: e.message}));
|
|
}
|
|
return;
|
|
}
|
|
|
|
|
|
// === Auto-Mastering (Matchering) ===
|
|
if (url.pathname === "/master" && req.method === "POST") {
|
|
const tmp_target = "/tmp/ai-master-target-" + Date.now() + ".wav";
|
|
const tmp_ref = "/tmp/ai-master-ref-" + Date.now() + ".wav";
|
|
const tmp_out = "/tmp/ai-master-out-" + Date.now() + ".wav";
|
|
try {
|
|
const body = JSON.parse(await readBody(req));
|
|
if (!body.target || !body.reference) throw new Error("target and reference base64 required");
|
|
fs.writeFileSync(tmp_target, Buffer.from(body.target, "base64"));
|
|
fs.writeFileSync(tmp_ref, Buffer.from(body.reference, "base64"));
|
|
execSync("/home/kxkm/ai/ace-step/venv/bin/python /home/kxkm/ai/ace-step/master.py " + tmp_target + " " + tmp_ref + " " + tmp_out, {timeout:120000});
|
|
const buf = fs.readFileSync(tmp_out);
|
|
fs.unlinkSync(tmp_target); fs.unlinkSync(tmp_ref); fs.unlinkSync(tmp_out);
|
|
res.writeHead(200,{"Content-Type":"audio/wav","Content-Length":buf.length}); res.end(buf);
|
|
} catch(e) {
|
|
try{fs.unlinkSync(tmp_target)}catch{} try{fs.unlinkSync(tmp_ref)}catch{} try{fs.unlinkSync(tmp_out)}catch{}
|
|
res.writeHead(500,{"Content-Type":"application/json"}); res.end(JSON.stringify({error:e.message}));
|
|
}
|
|
return;
|
|
}
|
|
|
|
// === Speech-to-Text (faster-whisper) ===
|
|
if (url.pathname === "/stt" && req.method === "POST") {
|
|
const tmp = "/tmp/stt-input-" + Date.now() + ".wav";
|
|
const pyScript = "/tmp/stt-run-" + Date.now() + ".py";
|
|
try {
|
|
const chunks = [];
|
|
await new Promise((resolve) => { req.on("data", d => chunks.push(d)); req.on("end", resolve); });
|
|
fs.writeFileSync(tmp, Buffer.concat(chunks));
|
|
fs.writeFileSync(pyScript, `from faster_whisper import WhisperModel\nm=WhisperModel("small",device="cpu",compute_type="int8")\nsegs,_=m.transcribe("${tmp}",language="fr")\nprint(" ".join(s.text for s in segs))`);
|
|
const result = execSync("/home/kxkm/venv/bin/python3 " + pyScript, {timeout:30000,encoding:"utf8"});
|
|
fs.unlinkSync(tmp); fs.unlinkSync(pyScript);
|
|
res.writeHead(200,{"Content-Type":"application/json"});
|
|
res.end(JSON.stringify({ok:true,text:result.trim()}));
|
|
} catch(e) {
|
|
try{fs.unlinkSync(tmp)}catch{} try{fs.unlinkSync(pyScript)}catch{}
|
|
res.writeHead(500,{"Content-Type":"application/json"});
|
|
res.end(JSON.stringify({error:e.message}));
|
|
}
|
|
return;
|
|
}
|
|
res.writeHead(404); res.end("Not found");
|
|
});
|
|
server.listen(PORT, () => console.log(`[ai-bridge] :${PORT} — instruments: drums, bass, pad, choir, fx, drone, grain, glitch, circus, honk, sound-design`));
|