✨(frontend) integrate rnnoise-wasm for noise suppression

Add the @timephy/rnnoise-wasm dependency and implement noise suppression
in the RnnNoiseProcessor.

src/frontend/package-lock.json

@@ -15,6 +15,7 @@
         "@react-aria/toast": "3.0.1",
         "@remixicon/react": "4.6.0",
         "@tanstack/react-query": "5.69.0",
+        "@timephy/rnnoise-wasm": "^1.0.0",
         "crisp-sdk-web": "1.0.25",
         "hoofd": "1.7.3",
         "i18next": "24.2.3",
@@ -3888,6 +3889,11 @@
         "react": "^18 || ^19"
       }
     },
+    "node_modules/@timephy/rnnoise-wasm": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@timephy/rnnoise-wasm/-/rnnoise-wasm-1.0.0.tgz",
+      "integrity": "sha512-zzRdFyALbhaNIuEo3LKazPWxabatbPxkaBrHzRjGDlVHX2dFklh68HUENHlvHkKsq+OOQSC1ag5sGPDMDgg2CA=="
+    },
     "node_modules/@ts-morph/common": {
       "version": "0.25.0",
       "resolved": "https://registry.npmjs.org/@ts-morph/common/-/common-0.25.0.tgz",

src/frontend/package.json

@@ -20,6 +20,7 @@
     "@react-aria/toast": "3.0.1",
     "@remixicon/react": "4.6.0",
     "@tanstack/react-query": "5.69.0",
+    "@timephy/rnnoise-wasm": "^1.0.0",
     "crisp-sdk-web": "1.0.25",
     "hoofd": "1.7.3",
     "i18next": "24.2.3",

src/frontend/src/features/rooms/components/Conference.tsx

@@ -17,6 +17,7 @@ import posthog from 'posthog-js'
 import { css } from '@/styled-system/css'
 import { LocalUserChoices } from '../routes/Room'
 import { BackgroundProcessorFactory } from '../livekit/components/blur'
+import { RnnNoiseProcessor } from '../livekit/components/denoise/RnnNoiseProcessor'
 
 export const Conference = ({
   roomId,
@@ -112,7 +113,9 @@ export const Conference = ({
           serverUrl={data?.livekit?.url}
           token={data?.livekit?.token}
           connect={true}
-          audio={userConfig.audioEnabled}
+          audio={userConfig.audioEnabled && {
+            processor: new RnnNoiseProcessor(),
+          }}
           video={
             userConfig.videoEnabled && {
               processor: BackgroundProcessorFactory.deserializeProcessor(

src/frontend/src/features/rooms/components/Join.tsx

@@ -3,7 +3,7 @@ import { usePreviewTracks } from '@livekit/components-react'
 import { css } from '@/styled-system/css'
 import { Screen } from '@/layout/Screen'
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
-import { LocalVideoTrack, Track } from 'livekit-client'
+import { LocalVideoTrack, LocalAudioTrack, Track, TrackProcessor } from 'livekit-client'
 import { H } from '@/primitives/H'
 import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleDevice'
 import { Field } from '@/primitives/Field'
@@ -18,6 +18,7 @@ import {
 } from '../livekit/components/effects/EffectsConfiguration'
 import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices'
 import { BackgroundProcessorFactory } from '../livekit/components/blur'
+import { RnnNoiseProcessor, AudioProcessorInterface } from '../livekit/components/denoise/RnnNoiseProcessor'
 import { isMobileBrowser } from '@livekit/components-core'
 import { fetchRoom } from '@/features/rooms/api/fetchRoom'
 import { keys } from '@/api/queryKeys'
@@ -137,6 +138,7 @@ export const Join = ({
       initialUserChoices.processorSerialized
     )
   )
+  const audioProcessor = useMemo(() => new RnnNoiseProcessor(), [])
 
   useEffect(() => {
     saveAudioInputDeviceId(audioDeviceId)
@@ -179,7 +181,7 @@ export const Join = ({
     () =>
       tracks?.filter(
         (track) => track.kind === Track.Kind.Audio
-      )[0] as LocalVideoTrack,
+      )[0] as LocalAudioTrack,
     [tracks]
   )
 
@@ -293,6 +295,14 @@ export const Join = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [videoTrack])
 
+  // Same for audio processor
+  useEffect(() => {
+    if (audioProcessor && audioTrack && !audioTrack.getProcessor()) {
+      audioTrack.setProcessor(audioProcessor as TrackProcessor<Track.Kind.Audio>)
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [audioTrack])
+
   const renderWaitingState = () => {
     switch (status) {
       case ApiLobbyStatus.TIMEOUT:

src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx

@@ -18,6 +18,7 @@ import {
   LocalVideoTrack,
   Track,
   VideoCaptureOptions,
+  AudioCaptureOptions,
 } from 'livekit-client'
 
 import { Shortcut } from '@/features/shortcuts/types'
@@ -28,6 +29,7 @@ import { ButtonRecipeProps } from '@/primitives/buttonRecipe'
 import { useEffect } from 'react'
 import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'
 import { BackgroundProcessorFactory } from '../blur'
+import { RnnNoiseProcessor } from '../denoise/RnnNoiseProcessor'
 
 export type ToggleSource = Exclude<
   Track.Source,
@@ -127,6 +129,18 @@ export const SelectToggleDevice = <T extends ToggleSource>({
       toggle(!trackProps.enabled, {
         processor: processor,
       } as VideoCaptureOptions)
+    } else if (props.source === Track.Source.Microphone) {
+      // Add noise reduction processor for audio tracks
+      const noiseProcessor = new RnnNoiseProcessor()
+      
+      const toggle = trackProps.toggle as (
+        forceState: boolean,
+        captureOptions: AudioCaptureOptions
+      ) => Promise<void>
+
+      toggle(!trackProps.enabled, {
+        processor: noiseProcessor,
+      } as AudioCaptureOptions)
     } else {
       trackProps.toggle()
     }

src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts

@@ -0,0 +1,69 @@
+import { Track, TrackProcessor, ProcessorOptions } from 'livekit-client'
+import { NoiseSuppressorWorklet_Name } from "@timephy/rnnoise-wasm"
+import NoiseSuppressorWorklet from "@timephy/rnnoise-wasm/NoiseSuppressorWorklet?worker&url"
+
+export interface AudioProcessorInterface extends TrackProcessor<Track.Kind.Audio> {
+    name: string
+}
+
+export class RnnNoiseProcessor implements AudioProcessorInterface {
+    name: string = 'noise-reduction'
+    processedTrack?: MediaStreamTrack
+
+    private source?: MediaStreamTrack
+    private audioContext?: AudioContext
+    private sourceNode?: MediaStreamAudioSourceNode
+    private destinationNode?: MediaStreamAudioDestinationNode
+    private noiseSuppressionNode?: AudioWorkletNode
+    constructor() {
+        // Initialize any configuration options here when needed
+    }
+
+    async init(opts: ProcessorOptions<Track.Kind.Audio>) {
+        if (!opts.track) {
+            throw new Error('Track is required for audio processing')
+        }
+
+        this.source = opts.track as MediaStreamTrack
+        
+        // Set up Web Audio API nodes
+        this.audioContext = new AudioContext()
+        await this.audioContext.audioWorklet.addModule(NoiseSuppressorWorklet)
+        this.sourceNode = this.audioContext.createMediaStreamSource(new MediaStream([this.source]))  
+        this.destinationNode = this.audioContext.createMediaStreamDestination()
+
+        this.noiseSuppressionNode = new AudioWorkletNode(this.audioContext, NoiseSuppressorWorklet_Name)
+
+        // Connect the audio processing chain
+        this.sourceNode
+            .connect(this.noiseSuppressionNode)
+            .connect(this.destinationNode)
+        
+        // Get the processed track
+        const tracks = this.destinationNode.stream.getAudioTracks()
+        if (tracks.length === 0) {
+            throw new Error('No audio tracks found for processing')
+        }
+        this.processedTrack = tracks[0]
+    }
+
+    async restart(opts: ProcessorOptions<Track.Kind.Audio>) {
+        await this.destroy()
+        return this.init(opts)
+    }
+
+    async destroy() {
+        // Clean up audio nodes and context
+        this.sourceNode?.disconnect()
+        this.noiseSuppressionNode?.disconnect()
+        this.destinationNode?.disconnect()
+        await this.audioContext?.close()
+        
+        this.sourceNode = undefined
+        this.destinationNode = undefined
+        this.audioContext = undefined
+        this.source = undefined
+        this.processedTrack = undefined
+        this.noiseSuppressionNode = undefined
+    }
+}