diff --git a/README.md b/README.md index dc627519..efd3297c 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Ces instruments tournent dans le browser via AudioWorklet — pas de serveur né | **Glitch** | Générateur de textures glitch | Buffer repeat, stutter, bit crush, downsample, reverse, pitch shift | | **Circus** | Orgue de barbarie / calliope | 4 registres (Flute 8', Flute 4', Principal, Mixture), tremolo, wobble, air | | **Honk** | Klaxon / sirène / corne | 3 modes (Klaxon, Siren, Horn), sweep paramétrique, harmoniques | -| **Magenta** | AI MIDI generator | Magenta.js browser-side (MelodyRNN, DrumsRNN, ImprovRNN), temperature, steps | +| **Magenta** | Local algorithmic MIDI generator | Browser-side heuristic generation modes (Melody, Drums, Improv), temperature, steps | ## Instruments AI-backed @@ -45,7 +45,7 @@ Ces instruments génèrent de l'audio via le serveur AI Bridge (`:8301`). | Instrument | Backend | Description | |---|---|---| -| **AceStep** | ACE-Step | Génération musicale IA par prompt (style + bpm + durée) | +| **AceStep** | ACE-Step / AI Bridge | Génération audio par prompt, import automatique en sample, local-first puis fallback bridge | | **KokoroTTS** | Kokoro | Synthèse vocale rapide — 12 voix (af_heart, am_adam, bf_emma...) | | **Piper** | Piper/Chatterbox | Synthèse vocale TTS avec personas | diff --git a/packages/app/studio/public/manuals/devices/instruments/ace-step.md b/packages/app/studio/public/manuals/devices/instruments/ace-step.md index 317457eb..97a7a7b1 100644 --- a/packages/app/studio/public/manuals/devices/instruments/ace-step.md +++ b/packages/app/studio/public/manuals/devices/instruments/ace-step.md @@ -1,6 +1,6 @@ # IA AceStep -An AI music generator that creates audio from text prompts. +An AI audio generator that turns text prompts into playable samples. **Track type: Audio** — generated samples appear as regions on the timeline. @@ -8,7 +8,7 @@ An AI music generator that creates audio from text prompts. ## 0. Overview -_IA AceStep_ generates music from text descriptions using the ACE-Step or MusicGen AI backend. The generated WAV is loaded as a playable sample. Use the AI panel to describe the music you want. +_IA AceStep_ generates music from text descriptions and imports the rendered WAV into the device as a playable sample. The editor prefers a local ACE-Step backend when available, then falls back to the shared AI Bridge workflow. Example uses: @@ -54,13 +54,19 @@ Select from 14 presets: ambient, electronic, experimental, classical, jazz, indu ### 2.3 Composer -Click to start generation. A progress bar shows the status. Generation takes 20-60 seconds depending on duration. +Click to start generation. The UI reports provider detection, rendering progress and sample import state. --- ## 3. Backend -Uses ACE-Step (port 9200) when available, falls back to MusicGen (port 9000) via the AI Bridge (port 8301). +Preferred path: +- Local ACE-Step backend (`127.0.0.1:9200`) + +Fallback path: +- AI Bridge (`:8301`), which itself can route to ACE-Step or other backends + +The resulting audio is imported into the project and attached to the device sample slot automatically. --- diff --git a/packages/app/studio/public/manuals/devices/instruments/magenta.md b/packages/app/studio/public/manuals/devices/instruments/magenta.md index 25da85e4..d36774a0 100644 --- a/packages/app/studio/public/manuals/devices/instruments/magenta.md +++ b/packages/app/studio/public/manuals/devices/instruments/magenta.md @@ -1,12 +1,12 @@ # IA Magenta -An algorithmic MIDI sequence generator with built-in polyphonic synthesizer. +A local algorithmic MIDI sequence generator with built-in polyphonic synthesizer. --- ## 0. Overview -_IA Magenta_ generates MIDI note sequences using algorithmic composition and plays them through an integrated synthesizer. Three models offer different musical styles. No external connection required — all generation runs locally. +_IA Magenta_ generates MIDI note sequences using local algorithmic composition and plays them through an integrated synthesizer. Three generation modes offer different musical behaviors. No external connection is required. Example uses: @@ -35,12 +35,12 @@ Synth amplitude envelope. Low-pass filter frequency. -### 1.5 Model +### 1.5 Mode -Generation algorithm: -- **Melody** — minor scale patterns -- **Drums** — kick, snare, hihat, ride, crash, tom -- **Improv** — pentatonic with chromatic variations +Generation mode: +- **Melody** — melodic patterns +- **Drums** — simple drum hits and accents +- **Improv** — freer melodic motion with more variation ### 1.6 Temperature @@ -62,13 +62,17 @@ Stereo panning spread based on note pitch. ## 2. Generation -1. Set Model, Temperature, Steps and Density +1. Set Mode, Temperature, Steps and Density 2. Click **Generate** to create a sequence 3. Click **Play** to listen 4. Click **Stop** to halt playback MIDI notes also trigger the built-in synth directly. +## 3. Execution Model + +Generation is local and heuristic. This device does not depend on an external AI bridge or a remote model backend in its current form. + --- _IA instrument — openDIAW.be / KXKM_ diff --git a/packages/app/studio/src/service/AceStepGenerationService.ts b/packages/app/studio/src/service/AceStepGenerationService.ts new file mode 100644 index 00000000..52706d7f --- /dev/null +++ b/packages/app/studio/src/service/AceStepGenerationService.ts @@ -0,0 +1,145 @@ +export type AceStepGenerationState = + | "idle" + | "checking" + | "generating" + | "importing" + | "ready" + | "error" + +export type AceStepGenerationRequest = { + prompt: string + style: string + durationSec: number + bpm: number +} + +export type AceStepGenerationStatus = { + state: AceStepGenerationState + message: string + detail: string + progress: number + provider?: "ace-step-local" | "ai-bridge" +} + +export type AceStepGenerationResult = { + audio: Blob + durationSec: number + provider: "ace-step-local" | "ai-bridge" +} + +const DEFAULT_PROMPT = "ambient electronic" +const AI_BRIDGE = `${location.protocol === "https:" ? "https:" : "http:"}//${location.hostname}:8301` +const ACE_STEP_URL = "http://127.0.0.1:9200" + +export class AceStepGenerationService { + async generateMusic( + request: AceStepGenerationRequest, + report: (status: AceStepGenerationStatus) => void + ): Promise { + const prompt = request.prompt.trim() || DEFAULT_PROMPT + const durationSec = Math.round(request.durationSec) + + report({ + state: "checking", + message: "Verification du moteur local", + detail: "Recherche d'un backend ACE-Step local...", + progress: 10 + }) + + const local = await this.#generateFromLocal({ + prompt, + style: request.style, + durationSec, + bpm: request.bpm + }, report) + if (local !== null) { + return local + } + + report({ + state: "generating", + message: "Generation via AI Bridge", + detail: "Fallback vers le bridge audio partage.", + progress: 25, + provider: "ai-bridge" + }) + + const bridgeResponse = await fetch(`${AI_BRIDGE}/generate/music`, { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ + prompt: `${prompt}, ${request.style}`, + duration: durationSec, + style: request.style, + bpm: request.bpm + }), + signal: AbortSignal.timeout(300000) + }) + if (!bridgeResponse.ok) { + throw new Error(`AI Bridge HTTP ${bridgeResponse.status}`) + } + + report({ + state: "generating", + message: "Reception de l'audio", + detail: "Le rendu audio est pret a etre importe.", + progress: 90, + provider: "ai-bridge" + }) + + return { + audio: await bridgeResponse.blob(), + durationSec, + provider: "ai-bridge" + } + } + + async #generateFromLocal( + request: AceStepGenerationRequest, + report: (status: AceStepGenerationStatus) => void + ): Promise { + const health = await fetch(`${ACE_STEP_URL}/health`, { + signal: AbortSignal.timeout(2000) + }).then(async response => response.ok ? response.json() : null).catch(() => null) + if (health?.ok !== true) { + return null + } + + report({ + state: "generating", + message: "Generation locale", + detail: `ACE-Step local • ${request.bpm} BPM • ${request.durationSec}s`, + progress: 25, + provider: "ace-step-local" + }) + + const localResponse = await fetch(`${ACE_STEP_URL}/generate`, { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ + prompt: `${request.prompt}, ${request.style}`, + duration: request.durationSec, + steps: 8, + bpm: request.bpm + }), + signal: AbortSignal.timeout(300000) + }) + if (!localResponse.ok) { + throw new Error(`ACE-Step local HTTP ${localResponse.status}`) + } + + report({ + state: "generating", + message: "Reception de l'audio", + detail: "Le rendu local est pret a etre importe.", + progress: 90, + provider: "ace-step-local" + }) + + return { + audio: await localResponse.blob(), + durationSec: request.durationSec, + provider: "ace-step-local" + } + } +} diff --git a/packages/app/studio/src/service/StudioService.ts b/packages/app/studio/src/service/StudioService.ts index 5f432b01..b51a56a6 100644 --- a/packages/app/studio/src/service/StudioService.ts +++ b/packages/app/studio/src/service/StudioService.ts @@ -38,7 +38,7 @@ import {RouteLocation} from "@opendaw/lib-jsx" import {PPQN} from "@opendaw/lib-dsp" import {AnimationFrame, Browser, ConsoleCommands, Dragging, Files} from "@opendaw/lib-dom" import {Promises} from "@opendaw/lib-runtime" -import {ExportStemsConfiguration, InstrumentFactories, PresetDecoder} from "@opendaw/studio-adapters" +import {ExportStemsConfiguration, InstrumentFactories, PresetDecoder, Sample} from "@opendaw/studio-adapters" import {Address} from "@opendaw/lib-box" import { AudioContentFactory, @@ -71,6 +71,7 @@ import {SoftwareMIDIPanel} from "@/ui/software-midi/SoftwareMIDIPanel" import {Mixdowns} from "@/service/Mixdowns" import {ShadertoyState} from "@/ui/shadertoy/ShadertoyState" import {CodeEditorState} from "@/ui/werkstatt-editor/CodeEditorState" +import {AceStepGenerationService} from "@/service/AceStepGenerationService" /** * I am just piling stuff after stuff in here to boot the environment. @@ -106,6 +107,7 @@ export class StudioService implements ProjectEnv { readonly panelLayout = new PanelContents(createPanelFactory(this)) readonly spotlightDataSupplier = new SpotlightDataSupplier() readonly samplePlayback: SamplePlayback + readonly aceStepGeneration = new AceStepGenerationService() readonly recovery = new Recovery(() => this.#projectProfileService.getValue(), this) readonly engine = new EngineFacade() @@ -130,12 +132,12 @@ export class StudioService implements ProjectEnv { readonly cloudAuthManager: CloudAuthManager, readonly buildInfo: BuildInfo) { this.#sampleService = new SampleService(audioContext) - this.#sampleService.subscribe(([sample, _]) => this.#signals.notify({ + this.#sampleService.subscribe(([sample]) => this.#signals.notify({ type: "import-sample", sample })) this.#soundfontService = new SoundfontService() - this.#soundfontService.subscribe(([soundfont, _]) => this.#signals.notify({ + this.#soundfontService.subscribe(([soundfont]) => this.#signals.notify({ type: "import-soundfont", soundfont })) @@ -222,6 +224,11 @@ export class StudioService implements ProjectEnv { }) } + async importGeneratedSample(name: string, audio: Blob): Promise { + const arrayBuffer = await audio.arrayBuffer() + return this.#sampleService.importFile({name, arrayBuffer}) + } + async exportStems() { return this.#projectProfileService.getValue() .ifSome(async (profile) => { @@ -562,4 +569,4 @@ export class StudioService implements ProjectEnv { StudioPreferences.catchupAndSubscribe(value => document.body.classList.toggle("scrollbar-padding", value), "visibility", "scrollbar-padding") } -} \ No newline at end of file +} diff --git a/packages/app/studio/src/ui/dashboard/Dashboard.tsx b/packages/app/studio/src/ui/dashboard/Dashboard.tsx index 8fd84d80..2253e4d4 100644 --- a/packages/app/studio/src/ui/dashboard/Dashboard.tsx +++ b/packages/app/studio/src/ui/dashboard/Dashboard.tsx @@ -41,11 +41,11 @@ export const Dashboard = ({lifecycle, service}: Construct) => { href="https://github.com/andremichelle/openDAW" target="upstream">openDAW cree pour le spectacle 3615 J'ai pete de la compagnie KXKM. - Il ajoute 9 instruments AI (Drone, Grain, Glitch, Circus, Honk, Magenta, + Il ajoute 9 instruments IA / generatifs (Drone, Grain, Glitch, Circus, Honk, Magenta, AceStep, KokoroTTS, Piper) et un AI Bridge avec 19 backends audio.

- Instruments de clown | Generation IA | TTS 12 voix | MIDI Magenta.js | 19 backends audio + Instruments de clown | Generation locale et IA assistee | TTS 12 voix | MIDI algorithmique | 19 backends audio

diff --git a/packages/app/studio/src/ui/devices/instruments/AceStepDeviceEditor.sass b/packages/app/studio/src/ui/devices/instruments/AceStepDeviceEditor.sass index 83db29a0..7240a67e 100644 --- a/packages/app/studio/src/ui/devices/instruments/AceStepDeviceEditor.sass +++ b/packages/app/studio/src/ui/devices/instruments/AceStepDeviceEditor.sass @@ -1,9 +1,91 @@ @use "@/mixins" component - @include mixins.ControlLayout(1) + @include mixins.Control + display: grid + grid-template-columns: repeat(2, minmax(0, auto)) + gap: 0.5em + align-content: start - > div.sample-drop + > section.hero, + > section.section + background: rgba(white, 0.03) + box-shadow: inset 0 0 0 1px rgba(white, 0.04) + border-radius: 0.35em + padding: 0.45em + display: flex + flex-direction: column + gap: 0.35em + min-width: 0 + pointer-events: all + + > section.hero + grid-column: 1 / -1 + + > span.eyebrow + font-size: 0.45em + letter-spacing: 0.12em + text-transform: uppercase + color: rgba(white, 0.45) + + > h3 + margin: 0 + font-size: 0.9em + color: var(--color-text) + + > p + margin: 0 + font-size: 0.5em + line-height: 1.4 + color: rgba(white, 0.55) + + > section.compact + grid-column: span 1 + + > section.sample-section, + > section.ai-panel, + > section.note + grid-column: 1 / -1 + + > section.note > p + margin: 0 + font-size: 0.5em + line-height: 1.4 + color: rgba(white, 0.55) + + > section.section + > div.section-header + display: flex + flex-direction: column + gap: 0.15em + + > h4 + margin: 0 + font-size: 0.55em + text-transform: uppercase + letter-spacing: 0.08em + color: var(--color-cream) + + > span + font-size: 0.45em + color: rgba(white, 0.4) + + > div.control-row + display: flex + flex-wrap: wrap + gap: 0.25em + + label + display: flex + flex-direction: column + gap: 0.15em + font-size: 0.45em + color: rgba(white, 0.55) + + > section.ai-panel + min-width: 14em + + > section.sample-section > div.sample-drop border-radius: 50% color: var(--color-shadow) display: flex @@ -11,10 +93,12 @@ component align-items: center justify-content: center outline: 1px dashed rgba(white, 0.1) - margin: 0.375em position: relative cursor: pointer pointer-events: all + min-width: 3.25em + min-height: 3.25em + align-self: flex-start > svg width: 2em @@ -39,17 +123,14 @@ component color: var(--color-black) background-color: var(--color-blue) - > div.ai-panel - display: flex - flex-direction: column - gap: 0.25em - margin: 0.375em - padding: 0.25em - pointer-events: all - min-width: 10em - border-left: 1px solid rgba(white, 0.08) + > section.sample-section > span.detail + font-size: 0.4em + color: rgba(white, 0.45) + min-height: 1em - > textarea, > input[type="text"] + > section.ai-panel + textarea, + input[type="text"] font-size: 0.5em font-family: inherit width: 100% @@ -68,7 +149,7 @@ component &::placeholder color: rgba(white, 0.3) - > select + select font-size: 0.5em font-family: inherit padding: 0.2em 0.3em @@ -107,10 +188,23 @@ component opacity: 0.35 cursor: default - > span.status - font-size: 0.4em - color: rgba(white, 0.4) - min-height: 1em + > div.status-block + display: flex + flex-direction: column + gap: 0.1em + padding: 0.2em 0.25em + border-radius: 0.25em + background: rgba(0, 0, 0, 0.18) + + > span.status + font-size: 0.45em + color: var(--color-text) + min-height: 1em + + > span.detail + font-size: 0.4em + color: rgba(white, 0.45) + min-height: 1em > div.progress-bar height: 2px diff --git a/packages/app/studio/src/ui/devices/instruments/AceStepDeviceEditor.tsx b/packages/app/studio/src/ui/devices/instruments/AceStepDeviceEditor.tsx index c4acd5ec..84910214 100644 --- a/packages/app/studio/src/ui/devices/instruments/AceStepDeviceEditor.tsx +++ b/packages/app/studio/src/ui/devices/instruments/AceStepDeviceEditor.tsx @@ -1,5 +1,5 @@ import css from "./AceStepDeviceEditor.sass?inline" -import {asInstanceOf, Lifecycle} from "@opendaw/lib-std" +import {asInstanceOf, Lifecycle, UUID} from "@opendaw/lib-std" import {createElement} from "@opendaw/lib-jsx" import {DeviceEditor} from "@/ui/devices/DeviceEditor.tsx" import {MenuItems} from "@/ui/devices/menu-items.ts" @@ -12,10 +12,11 @@ import {AudioFileBox} from "@opendaw/studio-boxes" import {Icon} from "@/ui/components/Icon" import {SampleSelector, SampleSelectStrategy} from "@/ui/devices/SampleSelector" import {StudioService} from "@/service/StudioService" +import {AceStepGenerationState} from "@/service/AceStepGenerationService" const className = Html.adoptStyleSheet(css, "AceStepDeviceEditor") -const AI_BRIDGE = (location.protocol === "https:" ? "https:" : "http:") + "//" + location.hostname + ":8301" -const STYLES = ["ambient","electronic","experimental","classical","jazz","industrial","drone","glitch","techno","house","drum-n-bass","lo-fi","cinematic","world"] as const +const STYLES = ["ambient", "electronic", "experimental", "classical", "jazz", "industrial", "drone", "glitch", "techno", "house", "drum-n-bass", "lo-fi", "cinematic", "world"] as const +void createElement type Construct = { lifecycle: Lifecycle; service: StudioService; adapter: AceStepDeviceBoxAdapter; deviceHost: DeviceHost } @@ -25,63 +26,146 @@ export const AceStepDeviceEditor = ({lifecycle, service, adapter, deviceHost}: C const {editing, midiLearning} = project const knob = (parameter: typeof volume) => ControlBuilder.createKnob({lifecycle, editing, midiLearning, adapter, parameter}) const sampleDropZone: HTMLElement = (
) + const sampleMetaEl: HTMLElement = Aucun sample genere pour l'instant. const sampleSelector = new SampleSelector(service, SampleSelectStrategy.forPointerField(adapter.box.file)) lifecycle.ownAll( adapter.box.file.catchupAndSubscribe(pointer => pointer.targetVertex.match({ - none: () => sampleDropZone.removeAttribute("sample"), - some: ({box}) => sampleDropZone.setAttribute("sample", asInstanceOf(box, AudioFileBox).fileName.getValue()) + none: () => { + sampleDropZone.removeAttribute("sample") + sampleMetaEl.textContent = "Aucun sample genere pour l'instant." + }, + some: ({box}) => { + const fileName = asInstanceOf(box, AudioFileBox).fileName.getValue() + sampleDropZone.setAttribute("sample", fileName) + sampleMetaEl.textContent = `Cible actuelle: ${fileName}` + } })), sampleSelector.configureBrowseClick(sampleDropZone), sampleSelector.configureContextMenu(sampleDropZone), sampleSelector.configureDrop(sampleDropZone) ) + const promptArea: HTMLTextAreaElement =