From cf53e160b764159c8ca9bb717a72e83b1c800196 Mon Sep 17 00:00:00 2001 From: Arnaud Robin Date: Wed, 30 Apr 2025 11:20:30 +0200 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(frontend)=20integrate=20rnnoise-wasm?= =?UTF-8?q?=20for=20noise=20suppression?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the @timephy/rnnoise-wasm dependency and implement noise suppression in the RnnNoiseProcessor. --- src/frontend/package-lock.json | 6 ++++ src/frontend/package.json | 1 + .../components/denoise/RnnNoiseProcessor.ts | 28 +++++++++++-------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/frontend/package-lock.json b/src/frontend/package-lock.json index 826be5d2..1d43b03f 100644 --- a/src/frontend/package-lock.json +++ b/src/frontend/package-lock.json @@ -15,6 +15,7 @@ "@react-aria/toast": "3.0.1", "@remixicon/react": "4.6.0", "@tanstack/react-query": "5.69.0", + "@timephy/rnnoise-wasm": "^1.0.0", "crisp-sdk-web": "1.0.25", "hoofd": "1.7.3", "i18next": "24.2.3", @@ -3888,6 +3889,11 @@ "react": "^18 || ^19" } }, + "node_modules/@timephy/rnnoise-wasm": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@timephy/rnnoise-wasm/-/rnnoise-wasm-1.0.0.tgz", + "integrity": "sha512-zzRdFyALbhaNIuEo3LKazPWxabatbPxkaBrHzRjGDlVHX2dFklh68HUENHlvHkKsq+OOQSC1ag5sGPDMDgg2CA==" + }, "node_modules/@ts-morph/common": { "version": "0.25.0", "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.25.0.tgz", diff --git a/src/frontend/package.json b/src/frontend/package.json index f2a8efda..e98a527e 100644 --- a/src/frontend/package.json +++ b/src/frontend/package.json @@ -20,6 +20,7 @@ "@react-aria/toast": "3.0.1", "@remixicon/react": "4.6.0", "@tanstack/react-query": "5.69.0", + "@timephy/rnnoise-wasm": "^1.0.0", "crisp-sdk-web": "1.0.25", "hoofd": "1.7.3", "i18next": "24.2.3", diff --git a/src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts b/src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts index b9464d3b..a320daa8 100644 --- a/src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts @@ -1,13 +1,11 @@ import { Track, TrackProcessor, ProcessorOptions } from 'livekit-client' +import { NoiseSuppressorWorklet_Name } from "@timephy/rnnoise-wasm" +import NoiseSuppressorWorklet from "@timephy/rnnoise-wasm/NoiseSuppressorWorklet?worker&url" export interface AudioProcessorInterface extends TrackProcessor { name: string } -/** - * This processor will eventually handle noise reduction. - * Currently it's a pass-through processor that sets up the structure for future noise reduction implementation. - */ export class RnnNoiseProcessor implements AudioProcessorInterface { name: string = 'noise-reduction' processedTrack?: MediaStreamTrack @@ -16,7 +14,7 @@ export class RnnNoiseProcessor implements AudioProcessorInterface { private audioContext?: AudioContext private sourceNode?: MediaStreamAudioSourceNode private destinationNode?: MediaStreamAudioDestinationNode - + private noiseSuppressionNode?: AudioWorkletNode constructor() { // Initialize any configuration options here when needed } @@ -27,15 +25,20 @@ export class RnnNoiseProcessor implements AudioProcessorInterface { } this.source = opts.track as MediaStreamTrack - - // Set up basic Web Audio API nodes + + // Set up Web Audio API nodes this.audioContext = new AudioContext() - this.sourceNode = this.audioContext.createMediaStreamSource(new MediaStream([this.source])) + await this.audioContext.audioWorklet.addModule(NoiseSuppressorWorklet) + this.sourceNode = this.audioContext.createMediaStreamSource(new MediaStream([this.source])) this.destinationNode = this.audioContext.createMediaStreamDestination() - // For now, just connect source directly to destination (pass-through) - this.sourceNode.connect(this.destinationNode) + this.noiseSuppressionNode = new AudioWorkletNode(this.audioContext, NoiseSuppressorWorklet_Name) + // Connect the audio processing chain + this.sourceNode + .connect(this.noiseSuppressionNode) + .connect(this.destinationNode) + // Get the processed track const tracks = this.destinationNode.stream.getAudioTracks() if (tracks.length === 0) { @@ -52,12 +55,15 @@ export class RnnNoiseProcessor implements AudioProcessorInterface { async destroy() { // Clean up audio nodes and context this.sourceNode?.disconnect() + this.noiseSuppressionNode?.disconnect() + this.destinationNode?.disconnect() await this.audioContext?.close() - + this.sourceNode = undefined this.destinationNode = undefined this.audioContext = undefined this.source = undefined this.processedTrack = undefined + this.noiseSuppressionNode = undefined } }