diff --git a/src/frontend/src/features/rooms/components/Conference.tsx b/src/frontend/src/features/rooms/components/Conference.tsx index 73072500..89237680 100644 --- a/src/frontend/src/features/rooms/components/Conference.tsx +++ b/src/frontend/src/features/rooms/components/Conference.tsx @@ -17,6 +17,7 @@ import posthog from 'posthog-js' import { css } from '@/styled-system/css' import { LocalUserChoices } from '../routes/Room' import { BackgroundProcessorFactory } from '../livekit/components/blur' +import { RnnNoiseProcessor } from '../livekit/components/denoise/RnnNoiseProcessor' export const Conference = ({ roomId, @@ -112,7 +113,9 @@ export const Conference = ({ serverUrl={data?.livekit?.url} token={data?.livekit?.token} connect={true} - audio={userConfig.audioEnabled} + audio={userConfig.audioEnabled && { + processor: new RnnNoiseProcessor(), + }} video={ userConfig.videoEnabled && { processor: BackgroundProcessorFactory.deserializeProcessor( diff --git a/src/frontend/src/features/rooms/components/Join.tsx b/src/frontend/src/features/rooms/components/Join.tsx index 42095ed7..29d60f18 100644 --- a/src/frontend/src/features/rooms/components/Join.tsx +++ b/src/frontend/src/features/rooms/components/Join.tsx @@ -3,7 +3,7 @@ import { usePreviewTracks } from '@livekit/components-react' import { css } from '@/styled-system/css' import { Screen } from '@/layout/Screen' import { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { LocalVideoTrack, Track } from 'livekit-client' +import { LocalVideoTrack, LocalAudioTrack, Track, TrackProcessor } from 'livekit-client' import { H } from '@/primitives/H' import { SelectToggleDevice } from '../livekit/components/controls/SelectToggleDevice' import { Field } from '@/primitives/Field' @@ -18,6 +18,7 @@ import { } from '../livekit/components/effects/EffectsConfiguration' import { usePersistentUserChoices } from '../livekit/hooks/usePersistentUserChoices' import { BackgroundProcessorFactory } from '../livekit/components/blur' +import { RnnNoiseProcessor, AudioProcessorInterface } from '../livekit/components/denoise/RnnNoiseProcessor' import { isMobileBrowser } from '@livekit/components-core' import { fetchRoom } from '@/features/rooms/api/fetchRoom' import { keys } from '@/api/queryKeys' @@ -137,6 +138,7 @@ export const Join = ({ initialUserChoices.processorSerialized ) ) + const audioProcessor = useMemo(() => new RnnNoiseProcessor(), []) useEffect(() => { saveAudioInputDeviceId(audioDeviceId) @@ -179,7 +181,7 @@ export const Join = ({ () => tracks?.filter( (track) => track.kind === Track.Kind.Audio - )[0] as LocalVideoTrack, + )[0] as LocalAudioTrack, [tracks] ) @@ -293,6 +295,14 @@ export const Join = ({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [videoTrack]) + // Same for audio processor + useEffect(() => { + if (audioProcessor && audioTrack && !audioTrack.getProcessor()) { + audioTrack.setProcessor(audioProcessor as TrackProcessor) + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [audioTrack]) + const renderWaitingState = () => { switch (status) { case ApiLobbyStatus.TIMEOUT: diff --git a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx index 46dd442c..21d57ffd 100644 --- a/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx +++ b/src/frontend/src/features/rooms/livekit/components/controls/SelectToggleDevice.tsx @@ -18,6 +18,7 @@ import { LocalVideoTrack, Track, VideoCaptureOptions, + AudioCaptureOptions, } from 'livekit-client' import { Shortcut } from '@/features/shortcuts/types' @@ -28,6 +29,7 @@ import { ButtonRecipeProps } from '@/primitives/buttonRecipe' import { useEffect } from 'react' import { usePersistentUserChoices } from '../../hooks/usePersistentUserChoices' import { BackgroundProcessorFactory } from '../blur' +import { RnnNoiseProcessor } from '../denoise/RnnNoiseProcessor' export type ToggleSource = Exclude< Track.Source, @@ -127,6 +129,18 @@ export const SelectToggleDevice = ({ toggle(!trackProps.enabled, { processor: processor, } as VideoCaptureOptions) + } else if (props.source === Track.Source.Microphone) { + // Add noise reduction processor for audio tracks + const noiseProcessor = new RnnNoiseProcessor() + + const toggle = trackProps.toggle as ( + forceState: boolean, + captureOptions: AudioCaptureOptions + ) => Promise + + toggle(!trackProps.enabled, { + processor: noiseProcessor, + } as AudioCaptureOptions) } else { trackProps.toggle() } diff --git a/src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts b/src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts new file mode 100644 index 00000000..b9464d3b --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/components/denoise/RnnNoiseProcessor.ts @@ -0,0 +1,63 @@ +import { Track, TrackProcessor, ProcessorOptions } from 'livekit-client' + +export interface AudioProcessorInterface extends TrackProcessor { + name: string +} + +/** + * This processor will eventually handle noise reduction. + * Currently it's a pass-through processor that sets up the structure for future noise reduction implementation. + */ +export class RnnNoiseProcessor implements AudioProcessorInterface { + name: string = 'noise-reduction' + processedTrack?: MediaStreamTrack + + private source?: MediaStreamTrack + private audioContext?: AudioContext + private sourceNode?: MediaStreamAudioSourceNode + private destinationNode?: MediaStreamAudioDestinationNode + + constructor() { + // Initialize any configuration options here when needed + } + + async init(opts: ProcessorOptions) { + if (!opts.track) { + throw new Error('Track is required for audio processing') + } + + this.source = opts.track as MediaStreamTrack + + // Set up basic Web Audio API nodes + this.audioContext = new AudioContext() + this.sourceNode = this.audioContext.createMediaStreamSource(new MediaStream([this.source])) + this.destinationNode = this.audioContext.createMediaStreamDestination() + + // For now, just connect source directly to destination (pass-through) + this.sourceNode.connect(this.destinationNode) + + // Get the processed track + const tracks = this.destinationNode.stream.getAudioTracks() + if (tracks.length === 0) { + throw new Error('No audio tracks found for processing') + } + this.processedTrack = tracks[0] + } + + async restart(opts: ProcessorOptions) { + await this.destroy() + return this.init(opts) + } + + async destroy() { + // Clean up audio nodes and context + this.sourceNode?.disconnect() + await this.audioContext?.close() + + this.sourceNode = undefined + this.destinationNode = undefined + this.audioContext = undefined + this.source = undefined + this.processedTrack = undefined + } +}