✨(front) add posthog log for Firefox blurring

We want to be able to track device settings and versions when a user
enables this feature.

src/frontend/src/features/rooms/livekit/components/Effects.tsx

@@ -5,11 +5,9 @@ import { Text, P, ToggleButton, Div, H } from '@/primitives'
 import { useTranslation } from 'react-i18next'
 import { HStack, styled, VStack } from '@/styled-system/jsx'
 import {
-  BackgroundBlur,
-  BackgroundOptions,
-  ProcessorWrapper,
-  BackgroundTransformer,
-} from '@livekit/track-processors'
+  BackgroundBlurFactory,
+  BackgroundBlurProcessorInterface,
+} from './blur/index'
 
 const Information = styled('div', {
   base: {
@@ -27,6 +25,8 @@ enum BlurRadius {
   NORMAL = 10,
 }
 
+const isSupported = BackgroundBlurFactory.isSupported()
+
 export const Effects = () => {
   const { t } = useTranslation('rooms', { keyPrefix: 'effects' })
   const { isCameraEnabled, cameraTrack, localParticipant } =
@@ -37,15 +37,12 @@ export const Effects = () => {
   const localCameraTrack = cameraTrack?.track as LocalVideoTrack
 
   const getProcessor = () => {
-    return localCameraTrack?.getProcessor() as ProcessorWrapper<BackgroundOptions>
+    return localCameraTrack?.getProcessor() as BackgroundBlurProcessorInterface
   }
 
   const getBlurRadius = (): BlurRadius => {
     const processor = getProcessor()
-    return (
-      (processor?.transformer as BackgroundTransformer)?.options?.blurRadius ||
-      BlurRadius.NONE
-    )
+    return processor?.options.blurRadius || BlurRadius.NONE
   }
 
   const toggleBlur = async (blurRadius: number) => {
@@ -61,9 +58,11 @@ export const Effects = () => {
       if (blurRadius == currentBlurRadius && processor) {
         await localCameraTrack.stopProcessor()
       } else if (!processor) {
-        await localCameraTrack.setProcessor(BackgroundBlur(blurRadius))
+        await localCameraTrack.setProcessor(
+          BackgroundBlurFactory.getProcessor({ blurRadius })!
+        )
       } else {
-        await processor?.updateTransformerOptions({ blurRadius })
+        processor?.update({ blurRadius })
       }
     } catch (error) {
       console.error('Error applying blur:', error)
@@ -147,7 +146,7 @@ export const Effects = () => {
         >
           {t('heading')}
         </H>
-        {ProcessorWrapper.isSupported ? (
+        {isSupported ? (
           <HStack>
             <ToggleButton
               size={'sm'}

src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurCustomProcessor.ts

@@ -0,0 +1,281 @@
+import { ProcessorOptions, Track } from 'livekit-client'
+import posthog from 'posthog-js'
+import {
+  FilesetResolver,
+  ImageSegmenter,
+  ImageSegmenterResult,
+} from '@mediapipe/tasks-vision'
+import {
+  CLEAR_TIMEOUT,
+  SET_TIMEOUT,
+  TIMEOUT_TICK,
+  timerWorkerScript,
+} from './TimerWorker'
+import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'
+
+const PROCESSING_WIDTH = 256
+const PROCESSING_HEIGHT = 144
+
+const SEGMENTATION_MASK_CANVAS_ID = 'background-blur-local-segmentation'
+const BLUR_CANVAS_ID = 'background-blur-local'
+
+const DEFAULT_BLUR = '10'
+
+/**
+ * This implementation of video blurring is made to be run on CPU for browser that are
+ * not compatible with track-processor-js.
+ *
+ * It also make possible to run blurring on browser that does not implement MediaStreamTrackGenerator and
+ * MediaStreamTrackProcessor.
+ */
+export class BackgroundBlurCustomProcessor
+  implements BackgroundBlurProcessorInterface
+{
+  options: BackgroundOptions
+  name: string
+  processedTrack?: MediaStreamTrack | undefined
+
+  source?: MediaStreamTrack
+  sourceSettings?: MediaTrackSettings
+  videoElement?: HTMLVideoElement
+  videoElementLoaded?: boolean
+
+  // Canvas containg the video processing result, of which we extract as stream.
+  outputCanvas?: HTMLCanvasElement
+  outputCanvasCtx?: CanvasRenderingContext2D
+
+  imageSegmenter?: ImageSegmenter
+  imageSegmenterResult?: ImageSegmenterResult
+
+  // Canvas used for resizing video source and projecting mask.
+  segmentationMaskCanvas?: HTMLCanvasElement
+  segmentationMaskCanvasCtx?: CanvasRenderingContext2D
+
+  // Mask containg the inference result.
+  segmentationMask?: ImageData
+
+  // The resized image of the video source.
+  sourceImageData?: ImageData
+
+  timerWorker?: Worker
+
+  constructor(opts: BackgroundOptions) {
+    this.name = 'blur'
+    this.options = opts
+  }
+
+  static get isSupported() {
+    return navigator.userAgent.toLowerCase().includes('firefox')
+  }
+
+  async init(opts: ProcessorOptions<Track.Kind>) {
+    if (!opts.element) {
+      throw new Error('Element is required for processing')
+    }
+
+    this.source = opts.track as MediaStreamTrack
+    this.sourceSettings = this.source!.getSettings()
+    this.videoElement = opts.element as HTMLVideoElement
+
+    this._createMainCanvas()
+    this._createMaskCanvas()
+
+    const stream = this.outputCanvas!.captureStream()
+    const tracks = stream.getVideoTracks()
+    if (tracks.length == 0) {
+      throw new Error('No tracks found for processing')
+    }
+    this.processedTrack = tracks[0]
+
+    this.segmentationMask = new ImageData(PROCESSING_WIDTH, PROCESSING_HEIGHT)
+    await this.initSegmenter()
+    this._initWorker()
+
+    posthog.capture('firefox-blurring-init')
+  }
+
+  update(opts: BackgroundOptions): void {
+    this.options = opts
+  }
+
+  _initWorker() {
+    this.timerWorker = new Worker(timerWorkerScript, {
+      name: 'Blurring',
+    })
+    this.timerWorker.onmessage = (data) => this.onTimerMessage(data)
+    // When hiding camera then showing it again, the onloadeddata callback is not fired again.
+    if (this.videoElementLoaded) {
+      this.timerWorker!.postMessage({
+        id: SET_TIMEOUT,
+        timeMs: 1000 / 30,
+      })
+    } else {
+      this.videoElement!.onloadeddata = () => {
+        this.videoElementLoaded = true
+        this.timerWorker!.postMessage({
+          id: SET_TIMEOUT,
+          timeMs: 1000 / 30,
+        })
+      }
+    }
+  }
+
+  onTimerMessage(response: { data: { id: number } }) {
+    if (response.data.id === TIMEOUT_TICK) {
+      this.process()
+    }
+  }
+
+  async initSegmenter() {
+    const vision = await FilesetResolver.forVisionTasks(
+      'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
+    )
+    this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {
+      baseOptions: {
+        modelAssetPath:
+          'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite',
+        delegate: 'CPU', // Use CPU for Firefox.
+      },
+      runningMode: 'VIDEO',
+      outputCategoryMask: true,
+      outputConfidenceMasks: false,
+    })
+  }
+
+  /**
+   * Resize the source video to the processing resolution.
+   */
+  async sizeSource() {
+    this.segmentationMaskCanvasCtx?.drawImage(
+      this.videoElement!,
+      0,
+      0,
+      this.videoElement!.videoWidth,
+      this.videoElement!.videoHeight,
+      0,
+      0,
+      PROCESSING_WIDTH,
+      PROCESSING_HEIGHT
+    )
+
+    this.sourceImageData = this.segmentationMaskCanvasCtx?.getImageData(
+      0,
+      0,
+      PROCESSING_WIDTH,
+      PROCESSING_WIDTH
+    )
+  }
+
+  /**
+   * Run the segmentation.
+   */
+  async segment() {
+    const startTimeMs = performance.now()
+    return new Promise<void>((resolve) => {
+      this.imageSegmenter!.segmentForVideo(
+        this.sourceImageData!,
+        startTimeMs,
+        (result: ImageSegmenterResult) => {
+          this.imageSegmenterResult = result
+          resolve()
+        }
+      )
+    })
+  }
+
+  /**
+   * TODO: future improvement with WebGL.
+   */
+  async blur() {
+    const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
+    for (let i = 0; i < mask.length; ++i) {
+      this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
+    }
+
+    this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
+
+    this.outputCanvasCtx!.globalCompositeOperation = 'copy'
+    this.outputCanvasCtx!.filter = 'blur(8px)'
+
+    this.outputCanvasCtx!.drawImage(
+      this.segmentationMaskCanvas!,
+      0,
+      0,
+      PROCESSING_WIDTH,
+      PROCESSING_HEIGHT,
+      0,
+      0,
+      this.videoElement!.videoWidth,
+      this.videoElement!.videoHeight
+    )
+
+    this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
+    this.outputCanvasCtx!.filter = 'none'
+    this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
+
+    this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
+    this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)`
+    this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
+  }
+
+  async process() {
+    await this.sizeSource()
+    await this.segment()
+    await this.blur()
+    this.timerWorker!.postMessage({
+      id: SET_TIMEOUT,
+      timeMs: 1000 / 30,
+    })
+  }
+
+  _createMainCanvas() {
+    this.outputCanvas = document.querySelector(
+      'canvas#background-blur-local'
+    ) as HTMLCanvasElement
+    if (!this.outputCanvas) {
+      this.outputCanvas = this._createCanvas(
+        BLUR_CANVAS_ID,
+        this.sourceSettings!.width!,
+        this.sourceSettings!.height!
+      )
+    }
+    this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
+  }
+
+  _createMaskCanvas() {
+    this.segmentationMaskCanvas = document.querySelector(
+      `#${SEGMENTATION_MASK_CANVAS_ID}`
+    ) as HTMLCanvasElement
+    if (!this.segmentationMaskCanvas) {
+      this.segmentationMaskCanvas = this._createCanvas(
+        SEGMENTATION_MASK_CANVAS_ID,
+        PROCESSING_WIDTH,
+        PROCESSING_HEIGHT
+      )
+    }
+    this.segmentationMaskCanvasCtx =
+      this.segmentationMaskCanvas.getContext('2d')!
+  }
+
+  _createCanvas(id: string, width: number, height: number) {
+    const element = document.createElement('canvas')
+    element.setAttribute('id', id)
+    element.setAttribute('width', '' + width)
+    element.setAttribute('height', '' + height)
+    return element
+  }
+
+  async restart(opts: ProcessorOptions<Track.Kind>) {
+    await this.destroy()
+    return this.init(opts)
+  }
+
+  async destroy() {
+    this.timerWorker?.postMessage({
+      id: CLEAR_TIMEOUT,
+    })
+
+    this.timerWorker?.terminate()
+    this.imageSegmenter?.close()
+  }
+}

src/frontend/src/features/rooms/livekit/components/blur/BackgroundBlurTrackProcessorJsWrapper.ts

@@ -0,0 +1,48 @@
+import {
+  BackgroundBlur,
+  BackgroundTransformer,
+  ProcessorWrapper,
+} from '@livekit/track-processors'
+import { ProcessorOptions, Track } from 'livekit-client'
+import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'
+
+/**
+ * This is simply a wrapper around track-processor-js Processor
+ * in order to be compatible with a common interface BackgroundBlurProcessorInterface
+ * used accross the project.
+ */
+export class BackgroundBlurTrackProcessorJsWrapper
+  implements BackgroundBlurProcessorInterface
+{
+  name: string = 'Blur'
+
+  processor: ProcessorWrapper<BackgroundOptions>
+
+  constructor(opts: BackgroundOptions) {
+    this.processor = BackgroundBlur(opts.blurRadius)
+  }
+
+  async init(opts: ProcessorOptions<Track.Kind>) {
+    return this.processor.init(opts)
+  }
+
+  async restart(opts: ProcessorOptions<Track.Kind>) {
+    return this.processor.restart(opts)
+  }
+
+  async destroy() {
+    return this.processor.destroy()
+  }
+
+  update(opts: BackgroundOptions): void {
+    this.processor.updateTransformerOptions(opts)
+  }
+
+  get processedTrack() {
+    return this.processor.processedTrack
+  }
+
+  get options() {
+    return (this.processor.transformer as BackgroundTransformer).options
+  }
+}

src/frontend/src/features/rooms/livekit/components/blur/TimerWorker.ts

@@ -0,0 +1,71 @@
+/**
+ * From https://github.com/jitsi/jitsi-meet
+ */
+
+/**
+ * SET_TIMEOUT constant is used to set interval and it is set in
+ * the id property of the request.data property. TimeMs property must
+ * also be set.
+ *
+ * ```
+ * //Request.data example:
+ * {
+ *      id: SET_TIMEOUT,
+ *      timeMs: 33
+ * }
+ * ```
+ */
+export const SET_TIMEOUT = 1
+
+/**
+ * CLEAR_TIMEOUT constant is used to clear the interval and it is set in
+ * the id property of the request.data property.
+ *
+ * ```
+ * {
+ *      id: CLEAR_TIMEOUT
+ * }
+ * ```
+ */
+export const CLEAR_TIMEOUT = 2
+
+/**
+ * TIMEOUT_TICK constant is used as response and it is set in the id property.
+ *
+ * ```
+ * {
+ *      id: TIMEOUT_TICK
+ * }
+ * ```
+ */
+export const TIMEOUT_TICK = 3
+
+/**
+ * The following code is needed as string to create a URL from a Blob.
+ * The URL is then passed to a WebWorker. Reason for this is to enable
+ * use of setInterval that is not throttled when tab is inactive.
+ */
+const code = `
+    var timer;
+
+    onmessage = function(request) {
+        switch (request.data.id) {
+        case ${SET_TIMEOUT}: {
+            timer = setTimeout(() => {
+                postMessage({ id: ${TIMEOUT_TICK} });
+            }, request.data.timeMs);
+            break;
+        }
+        case ${CLEAR_TIMEOUT}: {
+            if (timer) {
+                clearTimeout(timer);
+            }
+            break;
+        }
+        }
+    };
+`
+
+export const timerWorkerScript = URL.createObjectURL(
+  new Blob([code], { type: 'application/javascript' })
+)

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -0,0 +1,32 @@
+import { ProcessorWrapper } from '@livekit/track-processors'
+import { Track, TrackProcessor } from 'livekit-client'
+import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'
+import { BackgroundBlurCustomProcessor } from './BackgroundBlurCustomProcessor'
+
+export type BackgroundOptions = {
+  blurRadius?: number
+}
+
+export interface BackgroundBlurProcessorInterface
+  extends TrackProcessor<Track.Kind> {
+  update(opts: BackgroundOptions): void
+  options: BackgroundOptions
+}
+
+export class BackgroundBlurFactory {
+  static isSupported() {
+    return (
+      ProcessorWrapper.isSupported || BackgroundBlurCustomProcessor.isSupported
+    )
+  }
+
+  static getProcessor(opts: BackgroundOptions) {
+    if (ProcessorWrapper.isSupported) {
+      return new BackgroundBlurTrackProcessorJsWrapper(opts)
+    }
+    if (BackgroundBlurCustomProcessor.isSupported) {
+      return new BackgroundBlurCustomProcessor(opts)
+    }
+    return null
+  }
+}

src/frontend/src/locales/en/rooms.json

@@ -87,7 +87,7 @@
   },
   "effects": {
     "activateCamera": "Your camera is disabled. Choose an option to enable it.",
-    "notAvailable": "The blur effect will be available soon on your browser. We're working on it! In the meantime, you can use Google Chrome :(",
+    "notAvailable": "The blur effect will be available soon on your browser. We're working on it! In the meantime, you can use Google Chrome for best performance or Firefox :(",
     "heading": "Blur",
     "blur": {
       "light": "Light blur",

src/frontend/src/locales/fr/rooms.json

@@ -87,7 +87,7 @@
   },
   "effects": {
     "activateCamera": "Votre camera est désactivée. Choisissez une option pour l'activer.",
-    "notAvailable": "L'effet de flou sera bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome :(",
+    "notAvailable": "L'effet de flou sera bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome pour une meilleure performance ou Firefox :(",
     "heading": "Flou",
     "blur": {
       "light": "Léger flou",