diff --git a/package-lock.json b/package-lock.json index fbd06008..13790876 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3097,9 +3097,9 @@ "license": "MIT" }, "node_modules/@types/node": { - "version": "25.2.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-25.2.0.tgz", - "integrity": "sha512-DZ8VwRFUNzuqJ5khrvwMXHmvPe+zGayJhr2CDNiKB1WBE1ST8Djl00D0IC4vvNmHMdj6DlbYRIaFE7WHjlDl5w==", + "version": "25.2.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-25.2.1.tgz", + "integrity": "sha512-CPrnr8voK8vC6eEtyRzvMpgp3VyVRhgclonE7qYi6P9sXwYb59ucfrnmFBTaP0yUi8Gk4yZg/LlTJULGxvTNsg==", "license": "MIT", "peer": true, "dependencies": { @@ -6025,9 +6025,9 @@ } }, "node_modules/force-graph": { - "version": "1.51.0", - "resolved": "https://registry.npmjs.org/force-graph/-/force-graph-1.51.0.tgz", - "integrity": "sha512-aTnihCmiMA0ItLJLCbrQYS9mzriopW24goFPgUnKAAmAlPogTSmFWqoBPMXzIfPb7bs04Hur5zEI4WYgLW3Sig==", + "version": "1.51.1", + "resolved": "https://registry.npmjs.org/force-graph/-/force-graph-1.51.1.tgz", + "integrity": "sha512-uEEX8iRzgq1IKRISOw6RrB2RLMhcI25xznQYrCTVvxZHZZ+A2jH6qIolYuwavVxAMi64pFp2yZm4KFVdD993cg==", "license": "MIT", "dependencies": { "@tweenjs/tween.js": "18 - 25", @@ -6383,9 +6383,9 @@ } }, "node_modules/get-tsconfig": { - "version": "4.13.1", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.1.tgz", - "integrity": "sha512-EoY1N2xCn44xU6750Sx7OjOIT59FkmstNc3X6y5xpz7D5cBtZRe/3pSlTkDJgqsOk3WwZPkWfonhhUJfttQo3w==", + "version": "4.13.3", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.3.tgz", + "integrity": "sha512-vp8Cj/+9Q/ibZUrq1rhy8mCTQpCk31A3uu9wc1C50yAb3x2pFHOsGdAZQ7jD86ARayyxZUViYeIztW+GE8dcrg==", "license": "MIT", "dependencies": { "resolve-pkg-maps": "^1.0.0" @@ -8327,9 +8327,9 @@ "license": "MIT" }, "node_modules/mediabunny": { - "version": "1.31.1", - "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.31.1.tgz", - "integrity": "sha512-7ijrySW675UTxXoAji8Xj/H8H1uyZhPDARlNO1FF1zGnzJV3snayU32c1bpumslx5vbUa/M7fm6kcG/3y7UxHQ==", + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.32.0.tgz", + "integrity": "sha512-wMWynAQqZkmsLm+d2+fovcBal3zBQu/bot3u+yV4bDTfNxPeEDHv1wjiz6yjyKcXBuxfaQK96gImqLcKLfJG7w==", "license": "MPL-2.0", "workspaces": [ "packages/*" @@ -12979,9 +12979,9 @@ } }, "packages/app/lab/node_modules/@types/node": { - "version": "22.19.8", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.8.tgz", - "integrity": "sha512-ebO/Yl+EAvVe8DnMfi+iaAyIqYdK0q/q0y0rw82INWEKJOBe6b/P3YWE8NW7oOlF/nXFNrHwhARrN/hdgDkraA==", + "version": "22.19.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.9.tgz", + "integrity": "sha512-PD03/U8g1F9T9MI+1OBisaIARhSzeidsUjQaf51fOxrfjeiKN9bLVO06lHuHYjxdnqLWJijJHfqXPSJri2EM2A==", "dev": true, "license": "MIT", "dependencies": { @@ -13075,9 +13075,9 @@ "license": "LGPL-3.0-or-later" }, "packages/app/nam-test/node_modules/@types/node": { - "version": "22.19.8", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.8.tgz", - "integrity": "sha512-ebO/Yl+EAvVe8DnMfi+iaAyIqYdK0q/q0y0rw82INWEKJOBe6b/P3YWE8NW7oOlF/nXFNrHwhARrN/hdgDkraA==", + "version": "22.19.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.9.tgz", + "integrity": "sha512-PD03/U8g1F9T9MI+1OBisaIARhSzeidsUjQaf51fOxrfjeiKN9bLVO06lHuHYjxdnqLWJijJHfqXPSJri2EM2A==", "dev": true, "license": "MIT", "dependencies": { @@ -13131,9 +13131,9 @@ } }, "packages/app/studio/node_modules/@types/node": { - "version": "22.19.8", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.8.tgz", - "integrity": "sha512-ebO/Yl+EAvVe8DnMfi+iaAyIqYdK0q/q0y0rw82INWEKJOBe6b/P3YWE8NW7oOlF/nXFNrHwhARrN/hdgDkraA==", + "version": "22.19.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.9.tgz", + "integrity": "sha512-PD03/U8g1F9T9MI+1OBisaIARhSzeidsUjQaf51fOxrfjeiKN9bLVO06lHuHYjxdnqLWJijJHfqXPSJri2EM2A==", "dev": true, "license": "MIT", "dependencies": { diff --git a/plans/monitoring-with-fx.md b/plans/monitoring-with-fx.md new file mode 100644 index 00000000..816f99a4 --- /dev/null +++ b/plans/monitoring-with-fx.md @@ -0,0 +1,253 @@ +# Monitoring with FX + +## Goal +Route monitoring audio through the AudioUnit's effect chain instead of directly to speakers. + +## Approach +1. Recreate `ChannelMergerNode` on every monitoring state change +2. Map: `Array<{ uuid: UUID.Bytes, channels: Array }>` +3. `MonitoringMixProcessor` reads assigned channels, mixes into instrument buffer +4. Map sent to worklet on every rebuild + +## Architecture + +### Main Thread (EngineWorklet.ts) +```typescript +#channelMerger: Nullable = null +#monitoringSources: Map = UUID.newMap() + +registerMonitoringSource(uuid: UUID.Bytes, node: AudioNode, numChannels: 1 | 2): void { + this.#monitoringSources.set(uuid, { node, numChannels }) + this.#rebuildMonitoringMerger() +} + +unregisterMonitoringSource(uuid: UUID.Bytes): void { + this.#monitoringSources.delete(uuid) + this.#rebuildMonitoringMerger() +} + +#rebuildMonitoringMerger(): void { + // Disconnect old merger + if (isDefined(this.#channelMerger)) { + this.#channelMerger.disconnect() + this.#channelMerger = null + } + + if (this.#monitoringSources.size === 0) { + this.#commands.updateMonitoringMap([]) + return + } + + // Calculate total channels needed + let totalChannels = 0 + for (const { numChannels } of this.#monitoringSources.values()) { + totalChannels += numChannels + } + + // Create new merger + this.#channelMerger = this.context.createChannelMerger(totalChannels) + this.#channelMerger.connect(this) + + // Connect sources and build map + const map: Array<{ uuid: UUID.Bytes, channels: Array }> = [] + let channel = 0 + + for (const [uuid, { node, numChannels }] of this.#monitoringSources) { + if (numChannels === 2) { + const splitter = this.context.createChannelSplitter(2) + node.connect(splitter) + splitter.connect(this.#channelMerger, 0, channel) + splitter.connect(this.#channelMerger, 1, channel + 1) + map.push({ uuid, channels: [channel, channel + 1] }) + channel += 2 + } else { + node.connect(this.#channelMerger, 0, channel) + map.push({ uuid, channels: [channel] }) + channel += 1 + } + } + + this.#commands.updateMonitoringMap(map) +} +``` + +### Worklet Thread (EngineProcessor.ts) +```typescript +#currentInput: ReadonlyArray = [] + +// In render() +render(inputs: Float32Array[][], ...): boolean { + this.#currentInput = inputs[0] ?? [] + // ... +} + +// Command handler +updateMonitoringMap: (map: Array<{ uuid: UUID.Bytes, channels: Array }>) => { + this.#audioUnits.forEach(unit => unit.clearMonitoringChannels()) + for (const { uuid, channels } of map) { + this.optAudioUnit(uuid).ifSome(unit => unit.setMonitoringChannels(channels)) + } +} + +getMonitoringChannel(channelIndex: int): Option { + if (channelIndex >= this.#currentInput.length) return Option.None + return Option.wrap(this.#currentInput[channelIndex]) +} +``` + +### MonitoringMixProcessor.ts +```typescript +export class MonitoringMixProcessor extends AbstractProcessor implements Processor, AudioInput { + #source: Option = Option.None + #channels: Option> = Option.None + + setAudioSource(source: AudioBuffer): Terminable { + this.#source = Option.wrap(source) + return { terminate: () => this.#source = Option.None } + } + + setChannels(channels: Array): void { + this.#channels = Option.wrap(channels) + } + + clearChannels(): void { + this.#channels = Option.None + } + + get isActive(): boolean { + return this.#channels.nonEmpty() + } + + process(_processInfo: ProcessInfo): void { + if (this.#source.isEmpty() || this.#channels.isEmpty()) return + const [targetL, targetR] = this.#source.unwrap().channels() + const channels = this.#channels.unwrap() + + const optL = this.context.getMonitoringChannel(channels[0]) + if (optL.isEmpty()) return + const inputL = optL.unwrap() + + const inputR = channels.length === 2 + ? this.context.getMonitoringChannel(channels[1]).unwrapOrElse(inputL) + : inputL + + for (let i = 0; i < RenderQuantum; i++) { + targetL[i] += inputL[i] + targetR[i] += inputR[i] + } + } +} +``` + +### AudioDeviceChain.ts (`#wire`) +```typescript +#monitoringMixer: Option = Option.None + +#wire(): void { + let source: AudioDeviceProcessor = optInput.unwrap() + let edgeSource: Processor = source.outgoing + + if (this.#monitoringMixer.nonEmpty() && this.#monitoringMixer.unwrap().isActive) { + const mixer = this.#monitoringMixer.unwrap() + this.#disconnector.own(mixer.setAudioSource(source.audioOutput)) + this.#disconnector.own(context.registerEdge(source.outgoing, mixer)) + edgeSource = mixer + } + + for (const target of this.#orderedEffects) { + if (target.adapter().enabledField.getValue()) { + this.#disconnector.own(target.setAudioSource(source.audioOutput)) + this.#disconnector.own(context.registerEdge(edgeSource, target.incoming)) + source = target + edgeSource = target.outgoing + } + } + // ... channelStrip, output +} +``` + +### CaptureAudio.ts +```typescript +type MonitoringMode = "off" | "direct" | "effects" + +#monitoringMode: MonitoringMode = "off" + +get monitoringMode(): MonitoringMode { return this.#monitoringMode } +set monitoringMode(value: MonitoringMode) { + if (this.#monitoringMode === value) return + this.#disconnectMonitoring() + this.#monitoringMode = value + this.#connectMonitoring() +} + +// In constructor, subscribe to requestChannels changes: +captureAudioBox.requestChannels.catchupAndSubscribe(owner => { + // ... existing channel handling ... + + // Re-register if monitoring with effects (channel count may have changed) + if (this.#monitoringMode === "effects" && isDefined(this.#audioChain)) { + const engine = this.manager.project.engine + engine.unregisterMonitoringSource(this.audioUnitBox.address.uuid) + engine.registerMonitoringSource(this.audioUnitBox.address.uuid, this.#audioChain.gainNode, this.#audioChain.channelCount) + } +}) + +#connectMonitoring(): void { + if (!isDefined(this.#audioChain)) return + switch (this.#monitoringMode) { + case "off": + break + case "direct": + this.#audioChain.gainNode.connect(this.manager.project.env.audioContext.destination) + break + case "effects": + const engine = this.manager.project.engine + engine.registerMonitoringSource(this.audioUnitBox.address.uuid, this.#audioChain.gainNode, this.#audioChain.channelCount) + break + } +} + +#disconnectMonitoring(): void { + if (!isDefined(this.#audioChain)) return + switch (this.#monitoringMode) { + case "off": + break + case "direct": + this.#audioChain.gainNode.disconnect(this.manager.project.env.audioContext.destination) + break + case "effects": + this.#audioChain.gainNode.disconnect() + this.manager.project.engine.unregisterMonitoringSource(this.audioUnitBox.address.uuid) + break + } +} +``` + +## Flow +``` +CaptureAudio.monitorWithEffects = true + ↓ +engine.registerMonitoringSource(uuid, gainNode, numChannels) + ↓ +#rebuildMonitoringMerger() + ↓ +- disconnect old merger +- create new ChannelMergerNode(totalChannels) +- connect all sources +- build map with channel assignments +- commands.updateMonitoringMap(map) + ↓ +EngineProcessor: clear all → set channels for units in map + ↓ +AudioUnit.setMonitoringChannels() → invalidateWiring() + ↓ +#wire() includes mixer in edge chain +``` + +## Processing Order +``` +instrument.process() → mixer.process() → effect.process() + ↓ ↓ +writes to buffer reads from input[0][channels], + mixes into source buffer +```