✨(frontend) add audio track processor

Implement an AudioProcessorInterface to allow for the audio track
to be modified. The goal is to add noise reduction to the track.

src/frontend/src/features/rooms/components/Conference.tsx

@@ -17,6 +17,7 @@ import posthog from 'posthog-js'
 import { css } from '@/styled-system/css'
 import { LocalUserChoices } from '../routes/Room'
 import { BackgroundProcessorFactory } from '../livekit/components/blur'
+import { RnnNoiseProcessor } from '../livekit/components/denoise/RnnNoiseProcessor'
 
 export const Conference = ({
   roomId,
@@ -112,7 +113,9 @@ export const Conference = ({
           serverUrl={data?.livekit?.url}
           token={data?.livekit?.token}
           connect={true}
-          audio={userConfig.audioEnabled}
+          audio={userConfig.audioEnabled && {
+            processor: new RnnNoiseProcessor(),
+          }}
           video={
             userConfig.videoEnabled && {
               processor: BackgroundProcessorFactory.deserializeProcessor(

src/frontend/src/features/rooms/components/Join.tsx

@@ -3,7 +3,7 @@ import { usePreviewTracks } from '@livekit/components-react'
 import { css } from '@/styled-system/css'
 import { Screen } from '@/layout/Screen'
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
-import { LocalVideoTrack, Track } from 'livekit-client'
+import { LocalVideoTrack, LocalAudioTrack, Track, TrackProcessor } from 'livekit-client'
 import { H } from '@/primitives/H'
 import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleDevice'
 import { Field } from '@/primitives/Field'
@@ -18,6 +18,7 @@ import {
 } from '../livekit/components/effects/EffectsConfiguration'
 import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices'
 import { BackgroundProcessorFactory } from '../livekit/components/blur'
+import { RnnNoiseProcessor, AudioProcessorInterface } from '../livekit/components/denoise/RnnNoiseProcessor'
 import { isMobileBrowser } from '@livekit/components-core'
 import { fetchRoom } from '@/features/rooms/api/fetchRoom'
 import { keys } from '@/api/queryKeys'
@@ -137,6 +138,7 @@ export const Join = ({
       initialUserChoices.processorSerialized
     )
   )
+  const audioProcessor = useMemo(() => new RnnNoiseProcessor(), [])
 
   useEffect(() => {
     saveAudioInputDeviceId(audioDeviceId)
@@ -179,7 +181,7 @@ export const Join = ({
     () =>
       tracks?.filter(
         (track) => track.kind === Track.Kind.Audio
-      )[0] as LocalVideoTrack,
+      )[0] as LocalAudioTrack,
     [tracks]
   )
 
@@ -293,6 +295,14 @@ export const Join = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [videoTrack])
 
+  // Same for audio processor
+  useEffect(() => {
+    if (audioProcessor && audioTrack && !audioTrack.getProcessor()) {
+      audioTrack.setProcessor(audioProcessor as TrackProcessor<Track.Kind.Audio>)
+    }
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [audioTrack])
+
   const renderWaitingState = () => {
     switch (status) {
       case ApiLobbyStatus.TIMEOUT:

src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx

@@ -18,6 +18,7 @@ import {
   LocalVideoTrack,
   Track,
   VideoCaptureOptions,
+  AudioCaptureOptions,
 } from 'livekit-client'
 
 import { Shortcut } from '@/features/shortcuts/types'
@@ -28,6 +29,7 @@ import { ButtonRecipeProps } from '@/primitives/buttonRecipe'
 import { useEffect } from 'react'
 import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices'
 import { BackgroundProcessorFactory } from '../blur'
+import { RnnNoiseProcessor } from '../denoise/RnnNoiseProcessor'
 
 export type ToggleSource = Exclude<
   Track.Source,
@@ -127,6 +129,18 @@ export const SelectToggleDevice = <T extends ToggleSource>({
       toggle(!trackProps.enabled, {
         processor: processor,
       } as VideoCaptureOptions)
+    } else if (props.source === Track.Source.Microphone) {
+      // Add noise reduction processor for audio tracks
+      const noiseProcessor = new RnnNoiseProcessor()
+      
+      const toggle = trackProps.toggle as (
+        forceState: boolean,
+        captureOptions: AudioCaptureOptions
+      ) => Promise<void>
+
+      toggle(!trackProps.enabled, {
+        processor: noiseProcessor,
+      } as AudioCaptureOptions)
     } else {
       trackProps.toggle()
     }

src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts

@@ -0,0 +1,63 @@
+import { Track, TrackProcessor, ProcessorOptions } from 'livekit-client'
+
+export interface AudioProcessorInterface extends TrackProcessor<Track.Kind.Audio> {
+    name: string
+}
+
+/**
+ * This processor will eventually handle noise reduction.
+ * Currently it's a pass-through processor that sets up the structure for future noise reduction implementation.
+ */
+export class RnnNoiseProcessor implements AudioProcessorInterface {
+    name: string = 'noise-reduction'
+    processedTrack?: MediaStreamTrack
+
+    private source?: MediaStreamTrack
+    private audioContext?: AudioContext
+    private sourceNode?: MediaStreamAudioSourceNode
+    private destinationNode?: MediaStreamAudioDestinationNode
+
+    constructor() {
+        // Initialize any configuration options here when needed
+    }
+
+    async init(opts: ProcessorOptions<Track.Kind.Audio>) {
+        if (!opts.track) {
+            throw new Error('Track is required for audio processing')
+        }
+
+        this.source = opts.track as MediaStreamTrack
+
+        // Set up basic Web Audio API nodes
+        this.audioContext = new AudioContext()
+        this.sourceNode = this.audioContext.createMediaStreamSource(new MediaStream([this.source]))
+        this.destinationNode = this.audioContext.createMediaStreamDestination()
+
+        // For now, just connect source directly to destination (pass-through)
+        this.sourceNode.connect(this.destinationNode)
+
+        // Get the processed track
+        const tracks = this.destinationNode.stream.getAudioTracks()
+        if (tracks.length === 0) {
+            throw new Error('No audio tracks found for processing')
+        }
+        this.processedTrack = tracks[0]
+    }
+
+    async restart(opts: ProcessorOptions<Track.Kind.Audio>) {
+        await this.destroy()
+        return this.init(opts)
+    }
+
+    async destroy() {
+        // Clean up audio nodes and context
+        this.sourceNode?.disconnect()
+        await this.audioContext?.close()
+
+        this.sourceNode = undefined
+        this.destinationNode = undefined
+        this.audioContext = undefined
+        this.source = undefined
+        this.processedTrack = undefined
+    }
+}