From 9d6ee568747947551d0dd7b56c50768777f92470 Mon Sep 17 00:00:00 2001 From: kxkm Date: Sun, 5 Apr 2026 00:31:40 +0200 Subject: [PATCH] feat: harmonize voice generators and AI taxonomy --- .../manuals/devices/instruments/circus.md | 4 +- .../manuals/devices/instruments/drone.md | 4 +- .../manuals/devices/instruments/glitch.md | 4 +- .../manuals/devices/instruments/grain.md | 4 +- .../manuals/devices/instruments/honk.md | 4 +- .../app/studio/src/service/StudioService.ts | 2 + .../src/service/VoiceGenerationService.ts | 114 ++++++++++++++ .../app/studio/src/ui/dashboard/Dashboard.tsx | 5 +- .../instruments/KokoroTtsDeviceEditor.sass | 128 +++++++++++++--- .../instruments/KokoroTtsDeviceEditor.tsx | 143 ++++++++++++++---- .../instruments/PiperDeviceEditor.sass | 128 +++++++++++++--- .../devices/instruments/PiperDeviceEditor.tsx | 132 ++++++++++++---- .../src/factories/InstrumentFactories.ts | 10 +- 13 files changed, 570 insertions(+), 112 deletions(-) create mode 100644 packages/app/studio/src/service/VoiceGenerationService.ts diff --git a/packages/app/studio/public/manuals/devices/instruments/circus.md b/packages/app/studio/public/manuals/devices/instruments/circus.md index 2ce6c28e..b1caead3 100644 --- a/packages/app/studio/public/manuals/devices/instruments/circus.md +++ b/packages/app/studio/public/manuals/devices/instruments/circus.md @@ -6,7 +6,7 @@ A barrel organ / calliope synthesizer with pipe harmonics and mechanical wobble. ## 0. Overview -_IA Circus_ simulates a steam-powered barrel organ (orgue de barbarie). Additive synthesis generates organ pipe tones with configurable harmonic content. Tremolo and wobble add the characteristic mechanical instability. +_IA Circus_ is a local generative calliope and barrel-organ instrument. Additive synthesis generates organ pipe tones with configurable harmonic content. Tremolo and wobble add the characteristic mechanical instability. No external AI backend is required. Example uses: @@ -63,4 +63,4 @@ Fully chromatic — play chords and melodies. Each note triggers an independent --- -_IA instrument — openDIAW.be / KXKM clown performance_ +_Generative local instrument — openDIAW.be / KXKM clown performance_ diff --git a/packages/app/studio/public/manuals/devices/instruments/drone.md b/packages/app/studio/public/manuals/devices/instruments/drone.md index e1a8d0a6..09c537e2 100644 --- a/packages/app/studio/public/manuals/devices/instruments/drone.md +++ b/packages/app/studio/public/manuals/devices/instruments/drone.md @@ -6,7 +6,7 @@ A pad/drone synthesizer with unison oscillators, low-pass filter and LFO modulat ## 0. Overview -_IA Drone_ generates thick, evolving pads and drones. Multiple detuned oscillator voices create a wide stereo field. A low-pass filter shapes the tone, and an LFO adds movement. +_IA Drone_ is a local generative pad and drone synthesizer. Multiple detuned oscillator voices create a wide stereo field. A low-pass filter shapes the tone, and an LFO adds movement. No external AI backend is required. Example uses: @@ -71,4 +71,4 @@ Play any MIDI note to trigger a drone at that pitch. Multiple notes create polyp --- -_IA instrument — openDIAW.be / KXKM_ +_Generative local instrument — openDIAW.be / KXKM_ diff --git a/packages/app/studio/public/manuals/devices/instruments/glitch.md b/packages/app/studio/public/manuals/devices/instruments/glitch.md index 22ccb772..a36687b3 100644 --- a/packages/app/studio/public/manuals/devices/instruments/glitch.md +++ b/packages/app/studio/public/manuals/devices/instruments/glitch.md @@ -6,7 +6,7 @@ A glitch texture generator with buffer repeat, stutter, bit crush and downsample ## 0. Overview -_IA Glitch_ captures audio into a buffer and applies destructive effects: stuttering repeats, bit crushing, downsampling, pitch shifting, and random reversal. Designed for chaotic textures and transitions. +_IA Glitch_ is a local generative texture instrument. It captures audio into a buffer and applies destructive effects: stuttering repeats, bit crushing, downsampling, pitch shifting, and random reversal. It is designed for chaotic textures and transitions without relying on an external AI backend. Example uses: @@ -63,4 +63,4 @@ Note-on activates the glitch engine. Note-off stops it. The glitch runs continuo --- -_IA instrument — openDIAW.be / KXKM_ +_Generative local instrument — openDIAW.be / KXKM_ diff --git a/packages/app/studio/public/manuals/devices/instruments/grain.md b/packages/app/studio/public/manuals/devices/instruments/grain.md index 762a42a1..4644750a 100644 --- a/packages/app/studio/public/manuals/devices/instruments/grain.md +++ b/packages/app/studio/public/manuals/devices/instruments/grain.md @@ -6,7 +6,7 @@ A granular synthesizer that chops a loaded sample into tiny grains and scatters ## 0. Overview -_IA Grain_ takes a loaded audio file and slices it into micro-grains (5-500ms). These grains are played back at random positions and pitches, creating evolving textures from any source material. +_IA Grain_ is a local generative granular instrument. It takes a loaded audio file and slices it into micro-grains (5-500ms). These grains are played back at random positions and pitches, creating evolving textures from any source material without an external AI backend. Example uses: @@ -65,4 +65,4 @@ Each MIDI note triggers granular playback at a pitch ratio relative to C3 (middl --- -_IA instrument — openDIAW.be / KXKM_ +_Generative local instrument — openDIAW.be / KXKM_ diff --git a/packages/app/studio/public/manuals/devices/instruments/honk.md b/packages/app/studio/public/manuals/devices/instruments/honk.md index 1798884a..8b84d5b8 100644 --- a/packages/app/studio/public/manuals/devices/instruments/honk.md +++ b/packages/app/studio/public/manuals/devices/instruments/honk.md @@ -6,7 +6,7 @@ A parametric klaxon, siren and horn with frequency sweep. ## 0. Overview -_IA Honk_ generates klaxons, sirens and horns with configurable frequency sweep patterns. Three modes offer different sweep behaviors. The instrument is pitch-aware: MIDI notes shift the base frequency. +_IA Honk_ is a local generative horn, siren and klaxon instrument with configurable frequency sweep patterns. Three modes offer different sweep behaviors. The instrument is pitch-aware: MIDI notes shift the base frequency. No external AI backend is required. Example uses: @@ -61,4 +61,4 @@ Note-on triggers the honk. MIDI pitch shifts the base frequency. Note-off starts --- -_IA instrument — openDIAW.be / KXKM clown performance_ +_Generative local instrument — openDIAW.be / KXKM clown performance_ diff --git a/packages/app/studio/src/service/StudioService.ts b/packages/app/studio/src/service/StudioService.ts index b51a56a6..f772d8ff 100644 --- a/packages/app/studio/src/service/StudioService.ts +++ b/packages/app/studio/src/service/StudioService.ts @@ -72,6 +72,7 @@ import {Mixdowns} from "@/service/Mixdowns" import {ShadertoyState} from "@/ui/shadertoy/ShadertoyState" import {CodeEditorState} from "@/ui/werkstatt-editor/CodeEditorState" import {AceStepGenerationService} from "@/service/AceStepGenerationService" +import {VoiceGenerationService} from "@/service/VoiceGenerationService" /** * I am just piling stuff after stuff in here to boot the environment. @@ -108,6 +109,7 @@ export class StudioService implements ProjectEnv { readonly spotlightDataSupplier = new SpotlightDataSupplier() readonly samplePlayback: SamplePlayback readonly aceStepGeneration = new AceStepGenerationService() + readonly voiceGeneration = new VoiceGenerationService() readonly recovery = new Recovery(() => this.#projectProfileService.getValue(), this) readonly engine = new EngineFacade() diff --git a/packages/app/studio/src/service/VoiceGenerationService.ts b/packages/app/studio/src/service/VoiceGenerationService.ts new file mode 100644 index 00000000..591e8817 --- /dev/null +++ b/packages/app/studio/src/service/VoiceGenerationService.ts @@ -0,0 +1,114 @@ +export type VoiceGenerationState = + | "idle" + | "checking" + | "generating" + | "importing" + | "ready" + | "error" + +export type VoiceGenerationStatus = { + state: VoiceGenerationState + message: string + detail: string + progress: number + provider: "kokoro-fast" | "piper-persona" +} + +export type FastVoiceRequest = { + text: string + voice: string + speed: number +} + +export type PersonaVoiceRequest = { + text: string + persona: string +} + +export type VoiceGenerationResult = { + audio: Blob + provider: "kokoro-fast" | "piper-persona" +} + +const AI_BRIDGE = `${location.protocol === "https:" ? "https:" : "http:"}//${location.hostname}:8301` + +export class VoiceGenerationService { + async generateFastVoice( + request: FastVoiceRequest, + report: (status: VoiceGenerationStatus) => void + ): Promise { + report({ + state: "checking", + message: "Verification du provider", + detail: `Kokoro rapide • ${request.voice}`, + progress: 15, + provider: "kokoro-fast" + }) + + const response = await fetch(`${AI_BRIDGE}/generate/voice-fast`, { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ + text: request.text.trim() || "Bonjour", + voice: request.voice, + speed: request.speed + }), + signal: AbortSignal.timeout(30000) + }) + if (!response.ok) { + throw new Error(`AI Bridge HTTP ${response.status}`) + } + + report({ + state: "generating", + message: "Reception de l'audio", + detail: "La voix rapide est prete a etre importee.", + progress: 90, + provider: "kokoro-fast" + }) + + return { + audio: await response.blob(), + provider: "kokoro-fast" + } + } + + async generatePersonaVoice( + request: PersonaVoiceRequest, + report: (status: VoiceGenerationStatus) => void + ): Promise { + report({ + state: "checking", + message: "Verification du provider", + detail: `Piper persona • ${request.persona}`, + progress: 15, + provider: "piper-persona" + }) + + const response = await fetch(`${AI_BRIDGE}/generate/voice`, { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify({ + text: request.text.trim() || "Bonjour", + persona: request.persona + }), + signal: AbortSignal.timeout(30000) + }) + if (!response.ok) { + throw new Error(`AI Bridge HTTP ${response.status}`) + } + + report({ + state: "generating", + message: "Reception de l'audio", + detail: "La prise vocale est prete a etre importee.", + progress: 90, + provider: "piper-persona" + }) + + return { + audio: await response.blob(), + provider: "piper-persona" + } + } +} diff --git a/packages/app/studio/src/ui/dashboard/Dashboard.tsx b/packages/app/studio/src/ui/dashboard/Dashboard.tsx index 2253e4d4..9c9aaeb5 100644 --- a/packages/app/studio/src/ui/dashboard/Dashboard.tsx +++ b/packages/app/studio/src/ui/dashboard/Dashboard.tsx @@ -7,6 +7,7 @@ import {Resources} from "@/ui/dashboard/Resources" import {DemoProjects} from "@/ui/dashboard/DemoProjects" const className = Html.adoptStyleSheet(css, "Dashboard") +void createElement const QUOTES = [ "\"Le bruit est le son que les autres font.\" -- Pierre Schaeffer", @@ -41,11 +42,11 @@ export const Dashboard = ({lifecycle, service}: Construct) => { href="https://github.com/andremichelle/openDAW" target="upstream">openDAW cree pour le spectacle 3615 J'ai pete de la compagnie KXKM. - Il ajoute 9 instruments IA / generatifs (Drone, Grain, Glitch, Circus, Honk, Magenta, + Il ajoute 9 instruments generatifs et IA assistee (Drone, Grain, Glitch, Circus, Honk, Magenta, AceStep, KokoroTTS, Piper) et un AI Bridge avec 19 backends audio.

- Instruments de clown | Generation locale et IA assistee | TTS 12 voix | MIDI algorithmique | 19 backends audio + Instruments generatifs locaux | IA audio assistee | TTS 12 voix | MIDI algorithmique | 19 backends audio

diff --git a/packages/app/studio/src/ui/devices/instruments/KokoroTtsDeviceEditor.sass b/packages/app/studio/src/ui/devices/instruments/KokoroTtsDeviceEditor.sass index 83db29a0..0a06a298 100644 --- a/packages/app/studio/src/ui/devices/instruments/KokoroTtsDeviceEditor.sass +++ b/packages/app/studio/src/ui/devices/instruments/KokoroTtsDeviceEditor.sass @@ -1,9 +1,89 @@ @use "@/mixins" component - @include mixins.ControlLayout(1) + @include mixins.Control + display: grid + grid-template-columns: repeat(2, minmax(0, auto)) + gap: 0.5em + align-content: start - > div.sample-drop + > section.hero, + > section.section + background: rgba(white, 0.03) + box-shadow: inset 0 0 0 1px rgba(white, 0.04) + border-radius: 0.35em + padding: 0.45em + display: flex + flex-direction: column + gap: 0.35em + min-width: 0 + pointer-events: all + + > section.hero + grid-column: 1 / -1 + + > span.eyebrow + font-size: 0.45em + letter-spacing: 0.12em + text-transform: uppercase + color: rgba(white, 0.45) + + > h3 + margin: 0 + font-size: 0.9em + color: var(--color-text) + + > p + margin: 0 + font-size: 0.5em + line-height: 1.4 + color: rgba(white, 0.55) + + > section.compact, + > section.sample-section, + > section.ai-panel, + > section.note + grid-column: 1 / -1 + + > section.note > p + margin: 0 + font-size: 0.5em + line-height: 1.4 + color: rgba(white, 0.55) + + > section.section + > div.section-header + display: flex + flex-direction: column + gap: 0.15em + + > h4 + margin: 0 + font-size: 0.55em + text-transform: uppercase + letter-spacing: 0.08em + color: var(--color-cream) + + > span + font-size: 0.45em + color: rgba(white, 0.4) + + > div.control-row + display: flex + flex-wrap: wrap + gap: 0.25em + + label + display: flex + flex-direction: column + gap: 0.15em + font-size: 0.45em + color: rgba(white, 0.55) + + > section.ai-panel + min-width: 14em + + > section.sample-section > div.sample-drop border-radius: 50% color: var(--color-shadow) display: flex @@ -11,10 +91,12 @@ component align-items: center justify-content: center outline: 1px dashed rgba(white, 0.1) - margin: 0.375em position: relative cursor: pointer pointer-events: all + min-width: 3.25em + min-height: 3.25em + align-self: flex-start > svg width: 2em @@ -39,17 +121,14 @@ component color: var(--color-black) background-color: var(--color-blue) - > div.ai-panel - display: flex - flex-direction: column - gap: 0.25em - margin: 0.375em - padding: 0.25em - pointer-events: all - min-width: 10em - border-left: 1px solid rgba(white, 0.08) + > section.sample-section > span.detail + font-size: 0.4em + color: rgba(white, 0.45) + min-height: 1em - > textarea, > input[type="text"] + > section.ai-panel + textarea, + input[type="text"] font-size: 0.5em font-family: inherit width: 100% @@ -68,7 +147,7 @@ component &::placeholder color: rgba(white, 0.3) - > select + select font-size: 0.5em font-family: inherit padding: 0.2em 0.3em @@ -107,10 +186,23 @@ component opacity: 0.35 cursor: default - > span.status - font-size: 0.4em - color: rgba(white, 0.4) - min-height: 1em + > div.status-block + display: flex + flex-direction: column + gap: 0.1em + padding: 0.2em 0.25em + border-radius: 0.25em + background: rgba(0, 0, 0, 0.18) + + > span.status + font-size: 0.45em + color: var(--color-text) + min-height: 1em + + > span.detail + font-size: 0.4em + color: rgba(white, 0.45) + min-height: 1em > div.progress-bar height: 2px diff --git a/packages/app/studio/src/ui/devices/instruments/KokoroTtsDeviceEditor.tsx b/packages/app/studio/src/ui/devices/instruments/KokoroTtsDeviceEditor.tsx index 470b3553..46df268f 100644 --- a/packages/app/studio/src/ui/devices/instruments/KokoroTtsDeviceEditor.tsx +++ b/packages/app/studio/src/ui/devices/instruments/KokoroTtsDeviceEditor.tsx @@ -1,5 +1,5 @@ import css from "./KokoroTtsDeviceEditor.sass?inline" -import {asInstanceOf, Lifecycle} from "@opendaw/lib-std" +import {asInstanceOf, Lifecycle, UUID} from "@opendaw/lib-std" import {createElement} from "@opendaw/lib-jsx" import {DeviceEditor} from "@/ui/devices/DeviceEditor.tsx" import {MenuItems} from "@/ui/devices/menu-items.ts" @@ -7,14 +7,15 @@ import {DeviceHost, InstrumentFactories, KokoroTtsDeviceBoxAdapter} from "@opend import {IconSymbol} from "@opendaw/studio-enums" import {ControlBuilder} from "@/ui/devices/ControlBuilder.tsx" import {DevicePeakMeter} from "@/ui/devices/panel/DevicePeakMeter.tsx" -import {Html} from "@opendaw/lib-dom" +import {Events, Html} from "@opendaw/lib-dom" import {AudioFileBox} from "@opendaw/studio-boxes" import {Icon} from "@/ui/components/Icon" import {SampleSelector, SampleSelectStrategy} from "@/ui/devices/SampleSelector" import {StudioService} from "@/service/StudioService" +import {VoiceGenerationState} from "@/service/VoiceGenerationService" const className = Html.adoptStyleSheet(css, "KokoroTtsDeviceEditor") -const AI_BRIDGE = (location.protocol === "https:" ? "https:" : "http:") + "//" + location.hostname + ":8301" +void createElement const VOICES = [ {id: "ff_siwis", label: "Siwis (FR ♀)"}, {id: "af_heart", label: "Heart (EN ♀)"}, @@ -38,59 +39,141 @@ export const KokoroTtsDeviceEditor = ({lifecycle, service, adapter, deviceHost}: const {editing, midiLearning} = project const knob = (parameter: typeof volume) => ControlBuilder.createKnob({lifecycle, editing, midiLearning, adapter, parameter}) const sampleDropZone: HTMLElement = (
) + const sampleMetaEl: HTMLElement = Aucun sample vocal genere pour l'instant. const sampleSelector = new SampleSelector(service, SampleSelectStrategy.forPointerField(adapter.box.file)) lifecycle.ownAll( adapter.box.file.catchupAndSubscribe(pointer => pointer.targetVertex.match({ - none: () => sampleDropZone.removeAttribute("sample"), - some: ({box}) => sampleDropZone.setAttribute("sample", asInstanceOf(box, AudioFileBox).fileName.getValue()) + none: () => { + sampleDropZone.removeAttribute("sample") + sampleMetaEl.textContent = "Aucun sample vocal genere pour l'instant." + }, + some: ({box}) => { + const fileName = asInstanceOf(box, AudioFileBox).fileName.getValue() + sampleDropZone.setAttribute("sample", fileName) + sampleMetaEl.textContent = `Cible actuelle: ${fileName}` + } })), sampleSelector.configureBrowseClick(sampleDropZone), sampleSelector.configureContextMenu(sampleDropZone), sampleSelector.configureDrop(sampleDropZone) ) + const textArea: HTMLTextAreaElement =