feat: harmonize voice generators and AI taxonomy

This commit is contained in:
kxkm
2026-04-05 00:31:40 +02:00
parent 8d9eea6a99
commit 9d6ee56874
13 changed files with 570 additions and 112 deletions
@@ -6,7 +6,7 @@ A barrel organ / calliope synthesizer with pipe harmonics and mechanical wobble.
## 0. Overview ## 0. Overview
_IA Circus_ simulates a steam-powered barrel organ (orgue de barbarie). Additive synthesis generates organ pipe tones with configurable harmonic content. Tremolo and wobble add the characteristic mechanical instability. _IA Circus_ is a local generative calliope and barrel-organ instrument. Additive synthesis generates organ pipe tones with configurable harmonic content. Tremolo and wobble add the characteristic mechanical instability. No external AI backend is required.
Example uses: Example uses:
@@ -63,4 +63,4 @@ Fully chromatic — play chords and melodies. Each note triggers an independent
--- ---
_IA instrument — openDIAW.be / KXKM clown performance_ _Generative local instrument — openDIAW.be / KXKM clown performance_
@@ -6,7 +6,7 @@ A pad/drone synthesizer with unison oscillators, low-pass filter and LFO modulat
## 0. Overview ## 0. Overview
_IA Drone_ generates thick, evolving pads and drones. Multiple detuned oscillator voices create a wide stereo field. A low-pass filter shapes the tone, and an LFO adds movement. _IA Drone_ is a local generative pad and drone synthesizer. Multiple detuned oscillator voices create a wide stereo field. A low-pass filter shapes the tone, and an LFO adds movement. No external AI backend is required.
Example uses: Example uses:
@@ -71,4 +71,4 @@ Play any MIDI note to trigger a drone at that pitch. Multiple notes create polyp
--- ---
_IA instrument — openDIAW.be / KXKM_ _Generative local instrument — openDIAW.be / KXKM_
@@ -6,7 +6,7 @@ A glitch texture generator with buffer repeat, stutter, bit crush and downsample
## 0. Overview ## 0. Overview
_IA Glitch_ captures audio into a buffer and applies destructive effects: stuttering repeats, bit crushing, downsampling, pitch shifting, and random reversal. Designed for chaotic textures and transitions. _IA Glitch_ is a local generative texture instrument. It captures audio into a buffer and applies destructive effects: stuttering repeats, bit crushing, downsampling, pitch shifting, and random reversal. It is designed for chaotic textures and transitions without relying on an external AI backend.
Example uses: Example uses:
@@ -63,4 +63,4 @@ Note-on activates the glitch engine. Note-off stops it. The glitch runs continuo
--- ---
_IA instrument — openDIAW.be / KXKM_ _Generative local instrument — openDIAW.be / KXKM_
@@ -6,7 +6,7 @@ A granular synthesizer that chops a loaded sample into tiny grains and scatters
## 0. Overview ## 0. Overview
_IA Grain_ takes a loaded audio file and slices it into micro-grains (5-500ms). These grains are played back at random positions and pitches, creating evolving textures from any source material. _IA Grain_ is a local generative granular instrument. It takes a loaded audio file and slices it into micro-grains (5-500ms). These grains are played back at random positions and pitches, creating evolving textures from any source material without an external AI backend.
Example uses: Example uses:
@@ -65,4 +65,4 @@ Each MIDI note triggers granular playback at a pitch ratio relative to C3 (middl
--- ---
_IA instrument — openDIAW.be / KXKM_ _Generative local instrument — openDIAW.be / KXKM_
@@ -6,7 +6,7 @@ A parametric klaxon, siren and horn with frequency sweep.
## 0. Overview ## 0. Overview
_IA Honk_ generates klaxons, sirens and horns with configurable frequency sweep patterns. Three modes offer different sweep behaviors. The instrument is pitch-aware: MIDI notes shift the base frequency. _IA Honk_ is a local generative horn, siren and klaxon instrument with configurable frequency sweep patterns. Three modes offer different sweep behaviors. The instrument is pitch-aware: MIDI notes shift the base frequency. No external AI backend is required.
Example uses: Example uses:
@@ -61,4 +61,4 @@ Note-on triggers the honk. MIDI pitch shifts the base frequency. Note-off starts
--- ---
_IA instrument — openDIAW.be / KXKM clown performance_ _Generative local instrument — openDIAW.be / KXKM clown performance_
@@ -72,6 +72,7 @@ import {Mixdowns} from "@/service/Mixdowns"
import {ShadertoyState} from "@/ui/shadertoy/ShadertoyState" import {ShadertoyState} from "@/ui/shadertoy/ShadertoyState"
import {CodeEditorState} from "@/ui/werkstatt-editor/CodeEditorState" import {CodeEditorState} from "@/ui/werkstatt-editor/CodeEditorState"
import {AceStepGenerationService} from "@/service/AceStepGenerationService" import {AceStepGenerationService} from "@/service/AceStepGenerationService"
import {VoiceGenerationService} from "@/service/VoiceGenerationService"
/** /**
* I am just piling stuff after stuff in here to boot the environment. * I am just piling stuff after stuff in here to boot the environment.
@@ -108,6 +109,7 @@ export class StudioService implements ProjectEnv {
readonly spotlightDataSupplier = new SpotlightDataSupplier() readonly spotlightDataSupplier = new SpotlightDataSupplier()
readonly samplePlayback: SamplePlayback readonly samplePlayback: SamplePlayback
readonly aceStepGeneration = new AceStepGenerationService() readonly aceStepGeneration = new AceStepGenerationService()
readonly voiceGeneration = new VoiceGenerationService()
readonly recovery = new Recovery(() => this.#projectProfileService.getValue(), this) readonly recovery = new Recovery(() => this.#projectProfileService.getValue(), this)
readonly engine = new EngineFacade() readonly engine = new EngineFacade()
@@ -0,0 +1,114 @@
export type VoiceGenerationState =
| "idle"
| "checking"
| "generating"
| "importing"
| "ready"
| "error"
export type VoiceGenerationStatus = {
state: VoiceGenerationState
message: string
detail: string
progress: number
provider: "kokoro-fast" | "piper-persona"
}
export type FastVoiceRequest = {
text: string
voice: string
speed: number
}
export type PersonaVoiceRequest = {
text: string
persona: string
}
export type VoiceGenerationResult = {
audio: Blob
provider: "kokoro-fast" | "piper-persona"
}
const AI_BRIDGE = `${location.protocol === "https:" ? "https:" : "http:"}//${location.hostname}:8301`
export class VoiceGenerationService {
async generateFastVoice(
request: FastVoiceRequest,
report: (status: VoiceGenerationStatus) => void
): Promise<VoiceGenerationResult> {
report({
state: "checking",
message: "Verification du provider",
detail: `Kokoro rapide • ${request.voice}`,
progress: 15,
provider: "kokoro-fast"
})
const response = await fetch(`${AI_BRIDGE}/generate/voice-fast`, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify({
text: request.text.trim() || "Bonjour",
voice: request.voice,
speed: request.speed
}),
signal: AbortSignal.timeout(30000)
})
if (!response.ok) {
throw new Error(`AI Bridge HTTP ${response.status}`)
}
report({
state: "generating",
message: "Reception de l'audio",
detail: "La voix rapide est prete a etre importee.",
progress: 90,
provider: "kokoro-fast"
})
return {
audio: await response.blob(),
provider: "kokoro-fast"
}
}
async generatePersonaVoice(
request: PersonaVoiceRequest,
report: (status: VoiceGenerationStatus) => void
): Promise<VoiceGenerationResult> {
report({
state: "checking",
message: "Verification du provider",
detail: `Piper persona • ${request.persona}`,
progress: 15,
provider: "piper-persona"
})
const response = await fetch(`${AI_BRIDGE}/generate/voice`, {
method: "POST",
headers: {"Content-Type": "application/json"},
body: JSON.stringify({
text: request.text.trim() || "Bonjour",
persona: request.persona
}),
signal: AbortSignal.timeout(30000)
})
if (!response.ok) {
throw new Error(`AI Bridge HTTP ${response.status}`)
}
report({
state: "generating",
message: "Reception de l'audio",
detail: "La prise vocale est prete a etre importee.",
progress: 90,
provider: "piper-persona"
})
return {
audio: await response.blob(),
provider: "piper-persona"
}
}
}
@@ -7,6 +7,7 @@ import {Resources} from "@/ui/dashboard/Resources"
import {DemoProjects} from "@/ui/dashboard/DemoProjects" import {DemoProjects} from "@/ui/dashboard/DemoProjects"
const className = Html.adoptStyleSheet(css, "Dashboard") const className = Html.adoptStyleSheet(css, "Dashboard")
void createElement
const QUOTES = [ const QUOTES = [
"\"Le bruit est le son que les autres font.\" -- Pierre Schaeffer", "\"Le bruit est le son que les autres font.\" -- Pierre Schaeffer",
@@ -41,11 +42,11 @@ export const Dashboard = ({lifecycle, service}: Construct) => {
href="https://github.com/andremichelle/openDAW" target="upstream">openDAW</a> cree href="https://github.com/andremichelle/openDAW" target="upstream">openDAW</a> cree
pour le spectacle <em>3615 J'ai pete</em> de la compagnie <a pour le spectacle <em>3615 J'ai pete</em> de la compagnie <a
href="https://kxkm.net" target="kxkm">KXKM</a>. href="https://kxkm.net" target="kxkm">KXKM</a>.
Il ajoute <strong>9 instruments IA / generatifs</strong> (Drone, Grain, Glitch, Circus, Honk, Magenta, Il ajoute <strong>9 instruments generatifs et IA assistee</strong> (Drone, Grain, Glitch, Circus, Honk, Magenta,
AceStep, KokoroTTS, Piper) et un <strong>AI Bridge</strong> avec 19 backends audio. AceStep, KokoroTTS, Piper) et un <strong>AI Bridge</strong> avec 19 backends audio.
</p> </p>
<p style={{margin: "0.5em 0", fontSize: "11px"}}> <p style={{margin: "0.5em 0", fontSize: "11px"}}>
Instruments de clown | Generation locale et IA assistee | TTS 12 voix | MIDI algorithmique | 19 backends audio Instruments generatifs locaux | IA audio assistee | TTS 12 voix | MIDI algorithmique | 19 backends audio
</p> </p>
<div className="columns"> <div className="columns">
<DemoProjects lifecycle={lifecycle} service={service}/> <DemoProjects lifecycle={lifecycle} service={service}/>
@@ -1,9 +1,89 @@
@use "@/mixins" @use "@/mixins"
component component
@include mixins.ControlLayout(1) @include mixins.Control
display: grid
grid-template-columns: repeat(2, minmax(0, auto))
gap: 0.5em
align-content: start
> div.sample-drop > section.hero,
> section.section
background: rgba(white, 0.03)
box-shadow: inset 0 0 0 1px rgba(white, 0.04)
border-radius: 0.35em
padding: 0.45em
display: flex
flex-direction: column
gap: 0.35em
min-width: 0
pointer-events: all
> section.hero
grid-column: 1 / -1
> span.eyebrow
font-size: 0.45em
letter-spacing: 0.12em
text-transform: uppercase
color: rgba(white, 0.45)
> h3
margin: 0
font-size: 0.9em
color: var(--color-text)
> p
margin: 0
font-size: 0.5em
line-height: 1.4
color: rgba(white, 0.55)
> section.compact,
> section.sample-section,
> section.ai-panel,
> section.note
grid-column: 1 / -1
> section.note > p
margin: 0
font-size: 0.5em
line-height: 1.4
color: rgba(white, 0.55)
> section.section
> div.section-header
display: flex
flex-direction: column
gap: 0.15em
> h4
margin: 0
font-size: 0.55em
text-transform: uppercase
letter-spacing: 0.08em
color: var(--color-cream)
> span
font-size: 0.45em
color: rgba(white, 0.4)
> div.control-row
display: flex
flex-wrap: wrap
gap: 0.25em
label
display: flex
flex-direction: column
gap: 0.15em
font-size: 0.45em
color: rgba(white, 0.55)
> section.ai-panel
min-width: 14em
> section.sample-section > div.sample-drop
border-radius: 50% border-radius: 50%
color: var(--color-shadow) color: var(--color-shadow)
display: flex display: flex
@@ -11,10 +91,12 @@ component
align-items: center align-items: center
justify-content: center justify-content: center
outline: 1px dashed rgba(white, 0.1) outline: 1px dashed rgba(white, 0.1)
margin: 0.375em
position: relative position: relative
cursor: pointer cursor: pointer
pointer-events: all pointer-events: all
min-width: 3.25em
min-height: 3.25em
align-self: flex-start
> svg > svg
width: 2em width: 2em
@@ -39,17 +121,14 @@ component
color: var(--color-black) color: var(--color-black)
background-color: var(--color-blue) background-color: var(--color-blue)
> div.ai-panel > section.sample-section > span.detail
display: flex font-size: 0.4em
flex-direction: column color: rgba(white, 0.45)
gap: 0.25em min-height: 1em
margin: 0.375em
padding: 0.25em
pointer-events: all
min-width: 10em
border-left: 1px solid rgba(white, 0.08)
> textarea, > input[type="text"] > section.ai-panel
textarea,
input[type="text"]
font-size: 0.5em font-size: 0.5em
font-family: inherit font-family: inherit
width: 100% width: 100%
@@ -68,7 +147,7 @@ component
&::placeholder &::placeholder
color: rgba(white, 0.3) color: rgba(white, 0.3)
> select select
font-size: 0.5em font-size: 0.5em
font-family: inherit font-family: inherit
padding: 0.2em 0.3em padding: 0.2em 0.3em
@@ -107,10 +186,23 @@ component
opacity: 0.35 opacity: 0.35
cursor: default cursor: default
> span.status > div.status-block
font-size: 0.4em display: flex
color: rgba(white, 0.4) flex-direction: column
min-height: 1em gap: 0.1em
padding: 0.2em 0.25em
border-radius: 0.25em
background: rgba(0, 0, 0, 0.18)
> span.status
font-size: 0.45em
color: var(--color-text)
min-height: 1em
> span.detail
font-size: 0.4em
color: rgba(white, 0.45)
min-height: 1em
> div.progress-bar > div.progress-bar
height: 2px height: 2px
@@ -1,5 +1,5 @@
import css from "./KokoroTtsDeviceEditor.sass?inline" import css from "./KokoroTtsDeviceEditor.sass?inline"
import {asInstanceOf, Lifecycle} from "@opendaw/lib-std" import {asInstanceOf, Lifecycle, UUID} from "@opendaw/lib-std"
import {createElement} from "@opendaw/lib-jsx" import {createElement} from "@opendaw/lib-jsx"
import {DeviceEditor} from "@/ui/devices/DeviceEditor.tsx" import {DeviceEditor} from "@/ui/devices/DeviceEditor.tsx"
import {MenuItems} from "@/ui/devices/menu-items.ts" import {MenuItems} from "@/ui/devices/menu-items.ts"
@@ -7,14 +7,15 @@ import {DeviceHost, InstrumentFactories, KokoroTtsDeviceBoxAdapter} from "@opend
import {IconSymbol} from "@opendaw/studio-enums" import {IconSymbol} from "@opendaw/studio-enums"
import {ControlBuilder} from "@/ui/devices/ControlBuilder.tsx" import {ControlBuilder} from "@/ui/devices/ControlBuilder.tsx"
import {DevicePeakMeter} from "@/ui/devices/panel/DevicePeakMeter.tsx" import {DevicePeakMeter} from "@/ui/devices/panel/DevicePeakMeter.tsx"
import {Html} from "@opendaw/lib-dom" import {Events, Html} from "@opendaw/lib-dom"
import {AudioFileBox} from "@opendaw/studio-boxes" import {AudioFileBox} from "@opendaw/studio-boxes"
import {Icon} from "@/ui/components/Icon" import {Icon} from "@/ui/components/Icon"
import {SampleSelector, SampleSelectStrategy} from "@/ui/devices/SampleSelector" import {SampleSelector, SampleSelectStrategy} from "@/ui/devices/SampleSelector"
import {StudioService} from "@/service/StudioService" import {StudioService} from "@/service/StudioService"
import {VoiceGenerationState} from "@/service/VoiceGenerationService"
const className = Html.adoptStyleSheet(css, "KokoroTtsDeviceEditor") const className = Html.adoptStyleSheet(css, "KokoroTtsDeviceEditor")
const AI_BRIDGE = (location.protocol === "https:" ? "https:" : "http:") + "//" + location.hostname + ":8301" void createElement
const VOICES = [ const VOICES = [
{id: "ff_siwis", label: "Siwis (FR ♀)"}, {id: "ff_siwis", label: "Siwis (FR ♀)"},
{id: "af_heart", label: "Heart (EN ♀)"}, {id: "af_heart", label: "Heart (EN ♀)"},
@@ -38,59 +39,141 @@ export const KokoroTtsDeviceEditor = ({lifecycle, service, adapter, deviceHost}:
const {editing, midiLearning} = project const {editing, midiLearning} = project
const knob = (parameter: typeof volume) => ControlBuilder.createKnob({lifecycle, editing, midiLearning, adapter, parameter}) const knob = (parameter: typeof volume) => ControlBuilder.createKnob({lifecycle, editing, midiLearning, adapter, parameter})
const sampleDropZone: HTMLElement = (<div className="sample-drop"><Icon symbol={IconSymbol.Waveform}/></div>) const sampleDropZone: HTMLElement = (<div className="sample-drop"><Icon symbol={IconSymbol.Waveform}/></div>)
const sampleMetaEl: HTMLElement = <span className="detail">Aucun sample vocal genere pour l'instant.</span>
const sampleSelector = new SampleSelector(service, SampleSelectStrategy.forPointerField(adapter.box.file)) const sampleSelector = new SampleSelector(service, SampleSelectStrategy.forPointerField(adapter.box.file))
lifecycle.ownAll( lifecycle.ownAll(
adapter.box.file.catchupAndSubscribe(pointer => pointer.targetVertex.match({ adapter.box.file.catchupAndSubscribe(pointer => pointer.targetVertex.match({
none: () => sampleDropZone.removeAttribute("sample"), none: () => {
some: ({box}) => sampleDropZone.setAttribute("sample", asInstanceOf(box, AudioFileBox).fileName.getValue()) sampleDropZone.removeAttribute("sample")
sampleMetaEl.textContent = "Aucun sample vocal genere pour l'instant."
},
some: ({box}) => {
const fileName = asInstanceOf(box, AudioFileBox).fileName.getValue()
sampleDropZone.setAttribute("sample", fileName)
sampleMetaEl.textContent = `Cible actuelle: ${fileName}`
}
})), })),
sampleSelector.configureBrowseClick(sampleDropZone), sampleSelector.configureBrowseClick(sampleDropZone),
sampleSelector.configureContextMenu(sampleDropZone), sampleSelector.configureContextMenu(sampleDropZone),
sampleSelector.configureDrop(sampleDropZone) sampleSelector.configureDrop(sampleDropZone)
) )
const textArea: HTMLTextAreaElement = <textarea rows={3} placeholder="Texte a synthetiser..."/> as HTMLTextAreaElement const textArea: HTMLTextAreaElement = <textarea rows={3} placeholder="Texte a synthetiser..."/> as HTMLTextAreaElement
const voiceSelect: HTMLSelectElement = (<select>{VOICES.map(v => <option value={v.id}>{v.label}</option>)}</select>) as HTMLSelectElement const voiceSelect: HTMLSelectElement = (<select>{VOICES.map(voice => <option value={voice.id}>{voice.label}</option>)}</select>) as HTMLSelectElement
voiceSelect.value = VOICES[Math.round(voiceIndex.getValue())]?.id || "af_heart" const updateVoiceSelect = () => voiceSelect.value = VOICES[Math.round(voiceIndex.getValue())]?.id || "af_heart"
const statusEl: HTMLElement = <span className="status"/> updateVoiceSelect()
lifecycle.ownAll(
voiceIndex.catchupAndSubscribe(updateVoiceSelect),
Events.subscribe(voiceSelect, "change", () => {
const index = Math.max(0, VOICES.findIndex(({id}) => id === voiceSelect.value))
editing.modify(() => voiceIndex.setValue(index))
})
)
const statusEl: HTMLElement = <span className="status">Pret a synthetiser</span>
const detailEl: HTMLElement = <span className="detail">Selectionnez une voix puis lancez la synthese rapide.</span>
const progressEl: HTMLElement = <div className="progress-bar"><div className="progress-fill"/></div> const progressEl: HTMLElement = <div className="progress-bar"><div className="progress-fill"/></div>
const genBtn: HTMLButtonElement = <button>Synthetiser</button> as HTMLButtonElement const providerEl: HTMLElement = <span className="detail">Provider: en attente</span>
const genBtn: HTMLButtonElement = <button>Generer une voix</button> as HTMLButtonElement
let busy = false let busy = false
const fill = progressEl.querySelector(".progress-fill") as HTMLElement
const setUiState = (state: VoiceGenerationState, status: string, detail: string, progress: number, provider: string = "en attente") => {
statusEl.textContent = status
detailEl.textContent = detail
providerEl.textContent = `Provider: ${provider}`
fill.style.width = `${progress}%`
genBtn.classList.toggle("generating", state === "checking" || state === "generating" || state === "importing")
}
genBtn.addEventListener("click", async () => { genBtn.addEventListener("click", async () => {
if (busy) return if (busy) return
busy = true; genBtn.disabled = true; genBtn.classList.add("generating"); genBtn.textContent = "Synthese..." busy = true
statusEl.textContent = "" genBtn.disabled = true
const fill = progressEl.querySelector(".progress-fill") as HTMLElement genBtn.textContent = "Synthese..."
fill.style.width = "30%" setUiState("checking", "Generation en cours", VOICES[Math.round(voiceIndex.getValue())]?.label ?? voiceSelect.value, 15)
try { try {
const resp = await fetch(`${AI_BRIDGE}/generate/voice-fast`, { const generation = await service.voiceGeneration.generateFastVoice({
method: "POST", headers: {"Content-Type": "application/json"}, text: textArea.value,
body: JSON.stringify({text: textArea.value.trim() || "Bonjour", voice: voiceSelect.value, speed: speed.getValue()}), voice: voiceSelect.value,
signal: AbortSignal.timeout(30000) speed: speed.getValue()
}, update => {
setUiState(update.state, update.message, update.detail, update.progress, update.provider)
}) })
if (!resp.ok) throw new Error(`HTTP ${resp.status}`) setUiState("importing", "Import du sample", VOICES[Math.round(voiceIndex.getValue())]?.label ?? voiceSelect.value, 95, generation.provider)
fill.style.width = "90%" const sample = await service.importGeneratedSample(
const blob = await resp.blob() `Kokoro ${voiceSelect.value} ${new Date().toISOString().slice(11, 19).replace(/:/g, "-")}`,
statusEl.textContent = `OK — ${voiceSelect.value} (${(blob.size / 176400).toFixed(1)}s)` generation.audio
fill.style.width = "100%" )
service.project.trackUserCreatedSample(UUID.parse(sample.uuid))
sampleSelector.newSample(sample)
setUiState("ready", "Voix generee", `${voiceSelect.value} • ${sample.duration.toFixed(1)}s`, 100, generation.provider)
} catch (err) { } catch (err) {
statusEl.textContent = `Erreur: ${err instanceof Error ? err.message : "echec"}` setUiState("error", "Echec de la synthese", err instanceof Error ? err.message : "Generation interrompue", 0)
fill.style.width = "0%"
} finally { } finally {
busy = false; genBtn.disabled = false; genBtn.classList.remove("generating"); genBtn.textContent = "Synthetiser" busy = false
genBtn.disabled = false
genBtn.textContent = "Generer une voix"
setTimeout(() => { fill.style.width = "0%" }, 2000) setTimeout(() => { fill.style.width = "0%" }, 2000)
} }
}) })
return ( return (
<DeviceEditor lifecycle={lifecycle} project={project} adapter={adapter} <DeviceEditor lifecycle={lifecycle} project={project} adapter={adapter}
populateMenu={parent => MenuItems.forAudioUnitInput(parent, service, deviceHost)} populateMenu={parent => MenuItems.forAudioUnitInput(parent, service, deviceHost)}
populateControls={() => ( populateControls={() => (
<div className={className}> <div className={className}>
{knob(volume)}{knob(release)}{knob(speed)}{sampleDropZone} <section className="hero">
<div className="ai-panel"> <span className="eyebrow">Voix rapide</span>
{textArea}{voiceSelect} <h3>Kokoro TTS</h3>
<p>Preparation de texte, choix de voix et suivi de synthese dans une seule vue compacte.</p>
</section>
<section className="section compact">
<div className="section-header">
<h4>Sortie</h4>
<span>niveau, relache, vitesse</span>
</div>
<div className="control-row">
{knob(volume)}
{knob(release)}
{knob(speed)}
</div>
</section>
<section className="section sample-section">
<div className="section-header">
<h4>Destination audio</h4>
<span>sample charge ou depot glisse</span>
</div>
{sampleDropZone}
{sampleMetaEl}
</section>
<section className="section ai-panel">
<div className="section-header">
<h4>Script vocal</h4>
<span>texte, voix, provider et lancement</span>
</div>
<label>
<span>Texte</span>
{textArea}
</label>
<label>
<span>Voix</span>
{voiceSelect}
</label>
<div className="ai-controls">{genBtn}</div> <div className="ai-controls">{genBtn}</div>
{progressEl}{statusEl} {progressEl}
</div> <div className="status-block">
{statusEl}
{detailEl}
{providerEl}
</div>
</section>
<section className="section note">
<div className="section-header">
<h4>Usage</h4>
</div>
<p>La synthese rapide importe maintenant automatiquement un sample dans le device et expose explicitement le provider utilise.</p>
</section>
</div> </div>
)} )}
populateMeter={() => (<DevicePeakMeter lifecycle={lifecycle} receiver={project.liveStreamReceiver} address={adapter.address}/>)} populateMeter={() => (<DevicePeakMeter lifecycle={lifecycle} receiver={project.liveStreamReceiver} address={adapter.address}/>)}
@@ -1,9 +1,89 @@
@use "@/mixins" @use "@/mixins"
component component
@include mixins.ControlLayout(1) @include mixins.Control
display: grid
grid-template-columns: repeat(2, minmax(0, auto))
gap: 0.5em
align-content: start
> div.sample-drop > section.hero,
> section.section
background: rgba(white, 0.03)
box-shadow: inset 0 0 0 1px rgba(white, 0.04)
border-radius: 0.35em
padding: 0.45em
display: flex
flex-direction: column
gap: 0.35em
min-width: 0
pointer-events: all
> section.hero
grid-column: 1 / -1
> span.eyebrow
font-size: 0.45em
letter-spacing: 0.12em
text-transform: uppercase
color: rgba(white, 0.45)
> h3
margin: 0
font-size: 0.9em
color: var(--color-text)
> p
margin: 0
font-size: 0.5em
line-height: 1.4
color: rgba(white, 0.55)
> section.compact,
> section.sample-section,
> section.ai-panel,
> section.note
grid-column: 1 / -1
> section.note > p
margin: 0
font-size: 0.5em
line-height: 1.4
color: rgba(white, 0.55)
> section.section
> div.section-header
display: flex
flex-direction: column
gap: 0.15em
> h4
margin: 0
font-size: 0.55em
text-transform: uppercase
letter-spacing: 0.08em
color: var(--color-cream)
> span
font-size: 0.45em
color: rgba(white, 0.4)
> div.control-row
display: flex
flex-wrap: wrap
gap: 0.25em
label
display: flex
flex-direction: column
gap: 0.15em
font-size: 0.45em
color: rgba(white, 0.55)
> section.ai-panel
min-width: 14em
> section.sample-section > div.sample-drop
border-radius: 50% border-radius: 50%
color: var(--color-shadow) color: var(--color-shadow)
display: flex display: flex
@@ -11,10 +91,12 @@ component
align-items: center align-items: center
justify-content: center justify-content: center
outline: 1px dashed rgba(white, 0.1) outline: 1px dashed rgba(white, 0.1)
margin: 0.375em
position: relative position: relative
cursor: pointer cursor: pointer
pointer-events: all pointer-events: all
min-width: 3.25em
min-height: 3.25em
align-self: flex-start
> svg > svg
width: 2em width: 2em
@@ -39,17 +121,14 @@ component
color: var(--color-black) color: var(--color-black)
background-color: var(--color-blue) background-color: var(--color-blue)
> div.ai-panel > section.sample-section > span.detail
display: flex font-size: 0.4em
flex-direction: column color: rgba(white, 0.45)
gap: 0.25em min-height: 1em
margin: 0.375em
padding: 0.25em
pointer-events: all
min-width: 10em
border-left: 1px solid rgba(white, 0.08)
> textarea, > input[type="text"] > section.ai-panel
textarea,
input[type="text"]
font-size: 0.5em font-size: 0.5em
font-family: inherit font-family: inherit
width: 100% width: 100%
@@ -68,7 +147,7 @@ component
&::placeholder &::placeholder
color: rgba(white, 0.3) color: rgba(white, 0.3)
> select select
font-size: 0.5em font-size: 0.5em
font-family: inherit font-family: inherit
padding: 0.2em 0.3em padding: 0.2em 0.3em
@@ -107,10 +186,23 @@ component
opacity: 0.35 opacity: 0.35
cursor: default cursor: default
> span.status > div.status-block
font-size: 0.4em display: flex
color: rgba(white, 0.4) flex-direction: column
min-height: 1em gap: 0.1em
padding: 0.2em 0.25em
border-radius: 0.25em
background: rgba(0, 0, 0, 0.18)
> span.status
font-size: 0.45em
color: var(--color-text)
min-height: 1em
> span.detail
font-size: 0.4em
color: rgba(white, 0.45)
min-height: 1em
> div.progress-bar > div.progress-bar
height: 2px height: 2px
@@ -1,5 +1,5 @@
import css from "./PiperDeviceEditor.sass?inline" import css from "./PiperDeviceEditor.sass?inline"
import {asInstanceOf, Lifecycle} from "@opendaw/lib-std" import {asInstanceOf, Lifecycle, UUID} from "@opendaw/lib-std"
import {createElement} from "@opendaw/lib-jsx" import {createElement} from "@opendaw/lib-jsx"
import {DeviceEditor} from "@/ui/devices/DeviceEditor.tsx" import {DeviceEditor} from "@/ui/devices/DeviceEditor.tsx"
import {MenuItems} from "@/ui/devices/menu-items.ts" import {MenuItems} from "@/ui/devices/menu-items.ts"
@@ -12,10 +12,11 @@ import {AudioFileBox} from "@opendaw/studio-boxes"
import {Icon} from "@/ui/components/Icon" import {Icon} from "@/ui/components/Icon"
import {SampleSelector, SampleSelectStrategy} from "@/ui/devices/SampleSelector" import {SampleSelector, SampleSelectStrategy} from "@/ui/devices/SampleSelector"
import {StudioService} from "@/service/StudioService" import {StudioService} from "@/service/StudioService"
import {VoiceGenerationState} from "@/service/VoiceGenerationService"
const className = Html.adoptStyleSheet(css, "PiperDeviceEditor") const className = Html.adoptStyleSheet(css, "PiperDeviceEditor")
const AI_BRIDGE = (location.protocol === "https:" ? "https:" : "http:") + "//" + location.hostname + ":8301" void createElement
const PERSONAS = ["pharmacius","schaeffer","merzbow","cage","radigue","sunra","haraway","batty","deleuze","turing"] as const const PERSONAS = ["pharmacius", "schaeffer", "merzbow", "cage", "radigue", "sunra", "haraway", "batty", "deleuze", "turing"] as const
type Construct = { lifecycle: Lifecycle; service: StudioService; adapter: PiperDeviceBoxAdapter; deviceHost: DeviceHost } type Construct = { lifecycle: Lifecycle; service: StudioService; adapter: PiperDeviceBoxAdapter; deviceHost: DeviceHost }
@@ -25,58 +26,131 @@ export const PiperDeviceEditor = ({lifecycle, service, adapter, deviceHost}: Con
const {editing, midiLearning} = project const {editing, midiLearning} = project
const knob = (parameter: typeof volume) => ControlBuilder.createKnob({lifecycle, editing, midiLearning, adapter, parameter}) const knob = (parameter: typeof volume) => ControlBuilder.createKnob({lifecycle, editing, midiLearning, adapter, parameter})
const sampleDropZone: HTMLElement = (<div className="sample-drop"><Icon symbol={IconSymbol.Waveform}/></div>) const sampleDropZone: HTMLElement = (<div className="sample-drop"><Icon symbol={IconSymbol.Waveform}/></div>)
const sampleMetaEl: HTMLElement = <span className="detail">Aucun sample vocal genere pour l'instant.</span>
const sampleSelector = new SampleSelector(service, SampleSelectStrategy.forPointerField(adapter.box.file)) const sampleSelector = new SampleSelector(service, SampleSelectStrategy.forPointerField(adapter.box.file))
lifecycle.ownAll( lifecycle.ownAll(
adapter.box.file.catchupAndSubscribe(pointer => pointer.targetVertex.match({ adapter.box.file.catchupAndSubscribe(pointer => pointer.targetVertex.match({
none: () => sampleDropZone.removeAttribute("sample"), none: () => {
some: ({box}) => sampleDropZone.setAttribute("sample", asInstanceOf(box, AudioFileBox).fileName.getValue()) sampleDropZone.removeAttribute("sample")
sampleMetaEl.textContent = "Aucun sample vocal genere pour l'instant."
},
some: ({box}) => {
const fileName = asInstanceOf(box, AudioFileBox).fileName.getValue()
sampleDropZone.setAttribute("sample", fileName)
sampleMetaEl.textContent = `Cible actuelle: ${fileName}`
}
})), })),
sampleSelector.configureBrowseClick(sampleDropZone), sampleSelector.configureBrowseClick(sampleDropZone),
sampleSelector.configureContextMenu(sampleDropZone), sampleSelector.configureContextMenu(sampleDropZone),
sampleSelector.configureDrop(sampleDropZone) sampleSelector.configureDrop(sampleDropZone)
) )
const textArea: HTMLTextAreaElement = <textarea rows={3} placeholder="Texte a prononcer (Piper TTS)..."/> as HTMLTextAreaElement const textArea: HTMLTextAreaElement = <textarea rows={3} placeholder="Texte a prononcer (Piper TTS)..."/> as HTMLTextAreaElement
const personaSelect: HTMLSelectElement = (<select>{PERSONAS.map(p => <option value={p}>{p}</option>)}</select>) as HTMLSelectElement const personaSelect: HTMLSelectElement = (<select>{PERSONAS.map(persona => <option value={persona}>{persona}</option>)}</select>) as HTMLSelectElement
const statusEl: HTMLElement = <span className="status"/> personaSelect.value = PERSONAS[0]
const statusEl: HTMLElement = <span className="status">Pret a prononcer</span>
const detailEl: HTMLElement = <span className="detail">Choisissez une persona et lancez la generation.</span>
const progressEl: HTMLElement = <div className="progress-bar"><div className="progress-fill"/></div> const progressEl: HTMLElement = <div className="progress-bar"><div className="progress-fill"/></div>
const genBtn: HTMLButtonElement = <button>Prononcer</button> as HTMLButtonElement const providerEl: HTMLElement = <span className="detail">Provider: en attente</span>
const genBtn: HTMLButtonElement = <button>Generer une prise</button> as HTMLButtonElement
let busy = false let busy = false
const fill = progressEl.querySelector(".progress-fill") as HTMLElement
const setUiState = (state: VoiceGenerationState, status: string, detail: string, progress: number, provider: string = "en attente") => {
statusEl.textContent = status
detailEl.textContent = detail
providerEl.textContent = `Provider: ${provider}`
fill.style.width = `${progress}%`
genBtn.classList.toggle("generating", state === "checking" || state === "generating" || state === "importing")
}
genBtn.addEventListener("click", async () => { genBtn.addEventListener("click", async () => {
if (busy) return if (busy) return
busy = true; genBtn.disabled = true; genBtn.classList.add("generating"); genBtn.textContent = "Synthese..." busy = true
statusEl.textContent = "" genBtn.disabled = true
const fill = progressEl.querySelector(".progress-fill") as HTMLElement genBtn.textContent = "Synthese..."
fill.style.width = "30%" setUiState("checking", "Generation en cours", personaSelect.value, 15)
try { try {
const resp = await fetch(`${AI_BRIDGE}/generate/voice`, { const generation = await service.voiceGeneration.generatePersonaVoice({
method: "POST", headers: {"Content-Type": "application/json"}, text: textArea.value,
body: JSON.stringify({text: textArea.value.trim() || "Bonjour", persona: personaSelect.value}), persona: personaSelect.value
signal: AbortSignal.timeout(30000) }, update => {
setUiState(update.state, update.message, update.detail, update.progress, update.provider)
}) })
if (!resp.ok) throw new Error(`HTTP ${resp.status}`) setUiState("importing", "Import du sample", personaSelect.value, 95, generation.provider)
fill.style.width = "90%" const sample = await service.importGeneratedSample(
const blob = await resp.blob() `Piper ${personaSelect.value} ${new Date().toISOString().slice(11, 19).replace(/:/g, "-")}`,
statusEl.textContent = `OK — ${personaSelect.value} (${(blob.size / 176400).toFixed(1)}s)` generation.audio
fill.style.width = "100%" )
service.project.trackUserCreatedSample(UUID.parse(sample.uuid))
sampleSelector.newSample(sample)
setUiState("ready", "Prise terminee", `${personaSelect.value} • ${sample.duration.toFixed(1)}s`, 100, generation.provider)
} catch (err) { } catch (err) {
statusEl.textContent = `Erreur: ${err instanceof Error ? err.message : "echec"}` setUiState("error", "Echec de la synthese", err instanceof Error ? err.message : "Generation interrompue", 0)
fill.style.width = "0%"
} finally { } finally {
busy = false; genBtn.disabled = false; genBtn.classList.remove("generating"); genBtn.textContent = "Prononcer" busy = false
genBtn.disabled = false
genBtn.textContent = "Generer une prise"
setTimeout(() => { fill.style.width = "0%" }, 2000) setTimeout(() => { fill.style.width = "0%" }, 2000)
} }
}) })
return ( return (
<DeviceEditor lifecycle={lifecycle} project={project} adapter={adapter} <DeviceEditor lifecycle={lifecycle} project={project} adapter={adapter}
populateMenu={parent => MenuItems.forAudioUnitInput(parent, service, deviceHost)} populateMenu={parent => MenuItems.forAudioUnitInput(parent, service, deviceHost)}
populateControls={() => ( populateControls={() => (
<div className={className}> <div className={className}>
{knob(volume)}{knob(release)}{knob(speed)}{sampleDropZone} <section className="hero">
<div className="ai-panel"> <span className="eyebrow">Voix mise en scene</span>
{textArea}{personaSelect} <h3>Piper</h3>
<p>Concentrez la direction de jeu, la persona et le lancement dans un flux plus lisible.</p>
</section>
<section className="section compact">
<div className="section-header">
<h4>Sortie</h4>
<span>niveau, relache, vitesse</span>
</div>
<div className="control-row">
{knob(volume)}
{knob(release)}
{knob(speed)}
</div>
</section>
<section className="section sample-section">
<div className="section-header">
<h4>Destination audio</h4>
<span>sample charge ou depot glisse</span>
</div>
{sampleDropZone}
{sampleMetaEl}
</section>
<section className="section ai-panel">
<div className="section-header">
<h4>Direction vocale</h4>
<span>texte, persona, provider et lancement</span>
</div>
<label>
<span>Texte</span>
{textArea}
</label>
<label>
<span>Persona</span>
{personaSelect}
</label>
<div className="ai-controls">{genBtn}</div> <div className="ai-controls">{genBtn}</div>
{progressEl}{statusEl} {progressEl}
</div> <div className="status-block">
{statusEl}
{detailEl}
{providerEl}
</div>
</section>
<section className="section note">
<div className="section-header">
<h4>Usage</h4>
</div>
<p>La prise vocale importe maintenant automatiquement un sample dans le device et rend le provider utilise explicite.</p>
</section>
</div> </div>
)} )}
populateMeter={() => (<DevicePeakMeter lifecycle={lifecycle} receiver={project.liveStreamReceiver} address={adapter.address}/>)} populateMeter={() => (<DevicePeakMeter lifecycle={lifecycle} receiver={project.liveStreamReceiver} address={adapter.address}/>)}
@@ -218,7 +218,7 @@ export namespace InstrumentFactories {
export const Drone: InstrumentFactory<void, DroneDeviceBox> = { export const Drone: InstrumentFactory<void, DroneDeviceBox> = {
defaultName: "IA Drone", defaultName: "IA Drone",
defaultIcon: IconSymbol.Sine, defaultIcon: IconSymbol.Sine,
description: "[AI] Pad/drone synthesizer", description: "[Generative] Pad/drone synthesizer",
manualPage: DeviceManualUrls.Drone, manualPage: DeviceManualUrls.Drone,
trackType: TrackType.Notes, trackType: TrackType.Notes,
create: (boxGraph: BoxGraph, create: (boxGraph: BoxGraph,
@@ -236,7 +236,7 @@ export namespace InstrumentFactories {
export const Grain: InstrumentFactory<AudioFileBox, GrainDeviceBox> = { export const Grain: InstrumentFactory<AudioFileBox, GrainDeviceBox> = {
defaultName: "IA Grain", defaultName: "IA Grain",
defaultIcon: IconSymbol.Waveform, defaultIcon: IconSymbol.Waveform,
description: "[AI] Granular synthesizer", description: "[Generative] Granular synthesizer",
manualPage: DeviceManualUrls.Grain, manualPage: DeviceManualUrls.Grain,
trackType: TrackType.Notes, trackType: TrackType.Notes,
create: (boxGraph: BoxGraph, create: (boxGraph: BoxGraph,
@@ -268,7 +268,7 @@ export namespace InstrumentFactories {
export const Glitch: InstrumentFactory<void, GlitchDeviceBox> = { export const Glitch: InstrumentFactory<void, GlitchDeviceBox> = {
defaultName: "IA Glitch", defaultName: "IA Glitch",
defaultIcon: IconSymbol.Bug, defaultIcon: IconSymbol.Bug,
description: "[AI] Glitch texture generator", description: "[Generative] Glitch texture generator",
manualPage: DeviceManualUrls.Glitch, manualPage: DeviceManualUrls.Glitch,
trackType: TrackType.Notes, trackType: TrackType.Notes,
create: (boxGraph: BoxGraph, create: (boxGraph: BoxGraph,
@@ -286,7 +286,7 @@ export namespace InstrumentFactories {
export const Circus: InstrumentFactory<void, CircusDeviceBox> = { export const Circus: InstrumentFactory<void, CircusDeviceBox> = {
defaultName: "IA Circus", defaultName: "IA Circus",
defaultIcon: IconSymbol.Tone3000, defaultIcon: IconSymbol.Tone3000,
description: "[AI] Barrel organ / calliope", description: "[Generative] Barrel organ / calliope",
manualPage: DeviceManualUrls.Circus, manualPage: DeviceManualUrls.Circus,
trackType: TrackType.Notes, trackType: TrackType.Notes,
create: (boxGraph: BoxGraph, create: (boxGraph: BoxGraph,
@@ -304,7 +304,7 @@ export namespace InstrumentFactories {
export const Honk: InstrumentFactory<void, HonkDeviceBox> = { export const Honk: InstrumentFactory<void, HonkDeviceBox> = {
defaultName: "IA Honk", defaultName: "IA Honk",
defaultIcon: IconSymbol.Speaker, defaultIcon: IconSymbol.Speaker,
description: "[AI] Klaxon / siren / horn", description: "[Generative] Klaxon / siren / horn",
manualPage: DeviceManualUrls.Honk, manualPage: DeviceManualUrls.Honk,
trackType: TrackType.Notes, trackType: TrackType.Notes,
create: (boxGraph: BoxGraph, create: (boxGraph: BoxGraph,