wip ok

src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts

@@ -17,6 +17,9 @@ import {
   ProcessorType,
 } from '.'
 
+import { DrawingUtils } from './DrawingUtils'
+import { DrawingUtils2 } from '@/features/rooms/livekit/components/blur/DrawingUtils2.ts'
+
 const PROCESSING_WIDTH = 256
 const PROCESSING_HEIGHT = 144
 
@@ -25,6 +28,8 @@ const BLUR_CANVAS_ID = 'background-blur-local'
 
 const DEFAULT_BLUR = '10'
 
+export class Wip {}
+
 /**
  * This implementation of video blurring is made to be run on CPU for browser that are
  * not compatible with track-processor-js.
@@ -50,8 +55,11 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
   imageSegmenterResult?: ImageSegmenterResult
 
   // Canvas used for resizing video source and projecting mask.
-  segmentationMaskCanvas?: HTMLCanvasElement
-  segmentationMaskCanvasCtx?: CanvasRenderingContext2D
+  workCanvas?: HTMLCanvasElement
+  // segmentationMaskCanvasCtx?: CanvasRenderingContext2D
+  drawingUtils: any
+  segmentationMaskCanvasCtx: any
+  segmentationMaskCanvas: any
 
   // Mask containg the inference result.
   segmentationMask?: ImageData
@@ -90,10 +98,11 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
 
     this._initVirtualBackgroundImage()
     this._createMainCanvas()
-    this._createMaskCanvas()
+    this._createWorkCanvas()
 
     const stream = this.outputCanvas!.captureStream()
     const tracks = stream.getVideoTracks()
+
     if (tracks.length == 0) {
       throw new Error('No tracks found for processing')
     }
@@ -159,38 +168,16 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
       baseOptions: {
         modelAssetPath:
           'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite',
-        delegate: 'CPU', // Use CPU for Firefox.
+        // 'https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/1/deeplab_v3.tflite',
+        delegate: 'GPU', // Use CPU for Firefox.
       },
       runningMode: 'VIDEO',
       outputCategoryMask: true,
       outputConfidenceMasks: false,
+      canvas: this.workCanvas,
     })
   }
 
-  /**
-   * Resize the source video to the processing resolution.
-   */
-  async sizeSource() {
-    this.segmentationMaskCanvasCtx?.drawImage(
-      this.videoElement!,
-      0,
-      0,
-      this.videoElement!.videoWidth,
-      this.videoElement!.videoHeight,
-      0,
-      0,
-      PROCESSING_WIDTH,
-      PROCESSING_HEIGHT
-    )
-
-    this.sourceImageData = this.segmentationMaskCanvasCtx?.getImageData(
-      0,
-      0,
-      PROCESSING_WIDTH,
-      PROCESSING_WIDTH
-    )
-  }
-
   /**
    * Run the segmentation.
    */
@@ -198,7 +185,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
     const startTimeMs = performance.now()
     return new Promise<void>((resolve) => {
       this.imageSegmenter!.segmentForVideo(
-        this.sourceImageData!,
+        this.videoElement,
         startTimeMs,
         (result: ImageSegmenterResult) => {
           this.imageSegmenterResult = result
@@ -212,92 +199,26 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
    * TODO: future improvement with WebGL.
    */
   async blur() {
-    const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
-    for (let i = 0; i < mask.length; ++i) {
-      this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
-    }
+    // console.log('$$ blur')
 
-    this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
+    const legendColor = [[255, 197, 0, 255]]
 
-    this.outputCanvasCtx!.globalCompositeOperation = 'copy'
-    this.outputCanvasCtx!.filter = 'blur(8px)'
-
-    // Put opacity mask.
-    this.outputCanvasCtx!.drawImage(
-      this.segmentationMaskCanvas!,
-      0,
-      0,
-      PROCESSING_WIDTH,
-      PROCESSING_HEIGHT,
-      0,
-      0,
-      this.videoElement!.videoWidth,
-      this.videoElement!.videoHeight
-    )
-
-    // Draw clear body.
-    this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
-    this.outputCanvasCtx!.filter = 'none'
-    this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
-
-    // Draw blurry background.
-    this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
-    this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)`
-    this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
-  }
-
-  /**
-   * TODO: future improvement with WebGL.
-   */
-  async drawVirtualBackground() {
-    const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
-    for (let i = 0; i < mask.length; ++i) {
-      this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
-    }
-
-    this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
-
-    this.outputCanvasCtx!.globalCompositeOperation = 'copy'
-    this.outputCanvasCtx!.filter = 'blur(8px)'
-
-    // Put opacity mask.
-    this.outputCanvasCtx!.drawImage(
-      this.segmentationMaskCanvas!,
-      0,
-      0,
-      PROCESSING_WIDTH,
-      PROCESSING_HEIGHT,
-      0,
-      0,
-      this.videoElement!.videoWidth,
-      this.videoElement!.videoHeight
-    )
-
-    // Draw clear body.
-    this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
-    this.outputCanvasCtx!.filter = 'none'
-    this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
-
-    // Draw virtual background.
-    this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
-    this.outputCanvasCtx!.drawImage(
-      this.virtualBackgroundImage!,
-      0,
-      0,
-      this.outputCanvas!.width,
-      this.outputCanvas!.height
+    this.drawingUtils.applyBackgroundBlur(
+      this.imageSegmenterResult.categoryMask,
+      this.videoElement,
+      3.0
     )
   }
 
   async process() {
-    await this.sizeSource()
     await this.segment()
 
     if (this.options.blurRadius) {
       await this.blur()
-    } else {
-      await this.drawVirtualBackground()
     }
+    // } else {
+    //   await this.drawVirtualBackground()
+    // }
     this.timerWorker!.postMessage({
       id: SET_TIMEOUT,
       timeMs: 1000 / 30,
@@ -318,19 +239,33 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
     this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
   }
 
-  _createMaskCanvas() {
-    this.segmentationMaskCanvas = document.querySelector(
-      `#${SEGMENTATION_MASK_CANVAS_ID}`
-    ) as HTMLCanvasElement
-    if (!this.segmentationMaskCanvas) {
-      this.segmentationMaskCanvas = this._createCanvas(
-        SEGMENTATION_MASK_CANVAS_ID,
+  _createWorkCanvas() {
+    this.workCanvas = document.querySelector(`#workcanvas`)
+    if (!this.workCanvas) {
+      this.workCanvas = this._wip(
+        'workcanvas',
         PROCESSING_WIDTH,
         PROCESSING_HEIGHT
       )
     }
-    this.segmentationMaskCanvasCtx =
-      this.segmentationMaskCanvas.getContext('2d')!
+    this.drawingUtils = new DrawingUtils2(this.workCanvas.getContext('webgl2'))
+  }
+
+  _wip(id: string, width: number, height: number) {
+    const wip = document.querySelector(`#wip`)
+    const element = document.createElement('canvas')
+    element.setAttribute('id', id)
+    element.setAttribute('width', '' + width)
+    element.setAttribute('height', '' + height)
+    element.style.position = 'absolute'
+    element.style.top = '0'
+    element.style.left = '0'
+    element.style.maxWidth = '611px'
+    element.style.maxHeight = '305px'
+
+    wip.appendChild(element)
+
+    return element
   }
 
   _createCanvas(id: string, width: number, height: number) {

src/frontend/src/features/rooms/livekit/components/blur/DrawingUtils.ts

@@ -0,0 +1,158 @@
+import { MPMask } from '@mediapipe/tasks-vision'
+import { CategoryMaskShaderContext } from './wip'
+
+// Type definitions
+type RGBAColor = [number, number, number, number]
+type ImageSource =
+  | HTMLCanvasElement
+  | HTMLImageElement
+  | HTMLVideoElement
+  | ImageData
+
+type CategoryToColorMap = Map<number, RGBAColor> | RGBAColor[]
+
+export class DrawingUtils {
+  private categoryMaskShaderContext?: CategoryMaskShaderContext
+  private readonly contextWebGL?: WebGL2RenderingContext
+
+  /**
+   * Creates a new DrawingUtils class.
+   *
+   * @param gpuContext The WebGL canvas rendering context to render into. If
+   *     your Task is using a GPU delegate, the context must be obtained from
+   * its canvas (provided via `setOptions({ canvas: .. })`).
+   */
+  constructor(gpuContext: WebGL2RenderingContext)
+  /**
+   * Creates a new DrawingUtils class.
+   *
+   * @param cpuContext The 2D canvas rendering context to render into. If
+   *     you are rendering GPU data you must also provide `gpuContext` to allow
+   *     for data conversion.
+   * @param gpuContext A WebGL canvas that is used for GPU rendering and for
+   *     converting GPU to CPU data. If your Task is using a GPU delegate, the
+   *     context must be obtained from  its canvas (provided via
+   *     `setOptions({ canvas: .. })`).
+   */
+  constructor(
+    cpuContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+    gpuContext?: WebGL2RenderingContext
+  )
+  constructor(
+    cpuOrGpuGontext:
+      | CanvasRenderingContext2D
+      | OffscreenCanvasRenderingContext2D
+      | WebGL2RenderingContext,
+    gpuContext?: WebGL2RenderingContext
+  ) {
+    if (
+      (typeof CanvasRenderingContext2D !== 'undefined' &&
+        cpuOrGpuGontext instanceof CanvasRenderingContext2D) ||
+      cpuOrGpuGontext instanceof OffscreenCanvasRenderingContext2D
+    ) {
+      // this.context2d = cpuOrGpuGontext
+      this.contextWebGL = gpuContext
+    } else {
+      // If the first `if` statement is false, then the first argument must be a
+      // WebGL2RenderingContext, since CanvasRenderingContext2D can't be passed
+      // as the first argument. However, typescript isn't smart enough to infer
+      // this so we cast.
+      this.contextWebGL = cpuOrGpuGontext as WebGL2RenderingContext
+    }
+  }
+
+  private getWebGLRenderingContext(): WebGL2RenderingContext {
+    if (!this.contextWebGL) {
+      throw new Error(
+        'GPU rendering requested but WebGL2RenderingContext not provided.'
+      )
+    }
+    return this.contextWebGL
+  }
+
+  private getCategoryMaskShaderContext(): CategoryMaskShaderContext {
+    if (!this.categoryMaskShaderContext) {
+      this.categoryMaskShaderContext = new CategoryMaskShaderContext()
+    }
+    return this.categoryMaskShaderContext
+  }
+
+  /** Draws a category mask on a WebGL2RenderingContext2D. */
+  private drawCategoryMaskWebGL(
+    categoryTexture: WebGLTexture,
+    background: RGBAColor | ImageSource,
+    categoryToColorMap: Map<number, RGBAColor> | RGBAColor[]
+  ): void {
+    const shaderContext = this.getCategoryMaskShaderContext()
+    const gl = this.getWebGLRenderingContext()
+
+    const backgroundImage = Array.isArray(background)
+      ? new ImageData(new Uint8ClampedArray(background), 1, 1)
+      : background
+
+    shaderContext.run(gl, /* flipTexturesVertically= */ true, () => {
+      shaderContext.bindAndUploadTextures(
+        categoryTexture,
+        backgroundImage,
+        categoryToColorMap
+      )
+      gl.clearColor(0, 0, 0, 0)
+      gl.clear(gl.COLOR_BUFFER_BIT)
+      gl.drawArrays(gl.TRIANGLE_FAN, 0, 4)
+      shaderContext.unbindTextures()
+    })
+  }
+
+  /**
+   * Draws a category mask using the provided category-to-color mapping.
+   *
+   * @export
+   * @param mask A category mask that was returned from a segmentation task.
+   * @param categoryToColorMap A map that maps category indices to RGBA
+   *     values. You must specify a map entry for each category.
+   * @param background A color or image to use as the background. Defaults to
+   *     black.
+   */
+  drawCategoryMask(
+    mask: MPMask,
+    categoryToColorMap: Map<number, RGBAColor>,
+    background?: RGBAColor | ImageSource
+  ): void
+  /**
+   * Draws a category mask using the provided color array.
+   *
+   * @export
+   * @param mask A category mask that was returned from a segmentation task.
+   * @param categoryToColorMap An array that maps indices to RGBA values. The
+   *     array's indices must correspond to the category indices of the model
+   *     and an entry must be provided for each category.
+   * @param background A color or image to use as the background. Defaults to
+   *     black.
+   */
+  drawCategoryMask(
+    mask: MPMask,
+    categoryToColorMap: RGBAColor[],
+    background?: RGBAColor | ImageSource
+  ): void
+  /** @export */
+  drawCategoryMask(
+    mask: MPMask,
+    categoryToColorMap: CategoryToColorMap,
+    background: RGBAColor | ImageSource = [0, 0, 0, 255]
+  ): void {
+    this.drawCategoryMaskWebGL(
+      mask.getAsWebGLTexture(),
+      background,
+      categoryToColorMap
+    )
+  }
+
+  /**
+   * Frees all WebGL resources held by this class.
+   * @export
+   */
+  close(): void {
+    this.categoryMaskShaderContext?.close()
+    this.categoryMaskShaderContext = undefined
+  }
+}

src/frontend/src/features/rooms/livekit/components/blur/DrawingUtils2.ts

@@ -0,0 +1,120 @@
+import { MPMask } from '@mediapipe/tasks-vision'
+import { BackgroundBlurShaderContext } from './wip2.ts'
+
+// Type definitions
+type ImageSource =
+  | HTMLCanvasElement
+  | HTMLImageElement
+  | HTMLVideoElement
+  | ImageData
+
+export class DrawingUtils2 {
+  private backgroundBlurShaderContext?: BackgroundBlurShaderContext
+  private readonly contextWebGL?: WebGL2RenderingContext
+
+  /**
+   * Creates a new DrawingUtils class.
+   *
+   * @param gpuContext The WebGL canvas rendering context to render into. If
+   *     your Task is using a GPU delegate, the context must be obtained from
+   * its canvas (provided via `setOptions({ canvas: .. })`).
+   */
+  constructor(gpuContext: WebGL2RenderingContext)
+  /**
+   * Creates a new DrawingUtils class.
+   *
+   * @param cpuContext The 2D canvas rendering context to render into. If
+   *     you are rendering GPU data you must also provide `gpuContext` to allow
+   *     for data conversion.
+   * @param gpuContext A WebGL canvas that is used for GPU rendering and for
+   *     converting GPU to CPU data. If your Task is using a GPU delegate, the
+   *     context must be obtained from  its canvas (provided via
+   *     `setOptions({ canvas: .. })`).
+   */
+  constructor(
+    cpuContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+    gpuContext?: WebGL2RenderingContext
+  )
+  constructor(
+    cpuOrGpuContext:
+      | CanvasRenderingContext2D
+      | OffscreenCanvasRenderingContext2D
+      | WebGL2RenderingContext,
+    gpuContext?: WebGL2RenderingContext
+  ) {
+    if (
+      (typeof CanvasRenderingContext2D !== 'undefined' &&
+        cpuOrGpuContext instanceof CanvasRenderingContext2D) ||
+      cpuOrGpuContext instanceof OffscreenCanvasRenderingContext2D
+    ) {
+      this.contextWebGL = gpuContext
+    } else {
+      // If the first condition is false, then the first argument must be a
+      // WebGL2RenderingContext
+      this.contextWebGL = cpuOrGpuContext as WebGL2RenderingContext
+    }
+  }
+
+  private getWebGLRenderingContext(): WebGL2RenderingContext {
+    if (!this.contextWebGL) {
+      throw new Error(
+        'GPU rendering requested but WebGL2RenderingContext not provided.'
+      )
+    }
+    return this.contextWebGL
+  }
+
+  private getBackgroundBlurShaderContext(): BackgroundBlurShaderContext {
+    if (!this.backgroundBlurShaderContext) {
+      this.backgroundBlurShaderContext = new BackgroundBlurShaderContext()
+    }
+    return this.backgroundBlurShaderContext
+  }
+
+  /** Applies background blur effect using WebGL2. */
+  private applyBackgroundBlurWebGL(
+    maskTexture: WebGLTexture,
+    background: ImageSource,
+    blurAmount: number = 1.0
+  ): void {
+    const shaderContext = this.getBackgroundBlurShaderContext()
+    const gl = this.getWebGLRenderingContext()
+
+    shaderContext.run(gl, /* flipTexturesVertically= */ true, () => {
+      shaderContext.bindAndUploadTextures(maskTexture, background, blurAmount)
+      gl.clearColor(0, 0, 0, 0)
+      gl.clear(gl.COLOR_BUFFER_BIT)
+      gl.drawArrays(gl.TRIANGLE_FAN, 0, 4)
+      shaderContext.unbindTextures()
+    })
+  }
+
+  /**
+   * Applies a background blur effect using the mask to preserve the foreground.
+   *
+   * @export
+   * @param mask A binary mask that was returned from a segmentation task (1 for foreground, 0 for background).
+   * @param background The input image to blur the background of.
+   * @param blurAmount Optional blur amount parameter (defaults to 1.0, higher values create stronger blur).
+   */
+  applyBackgroundBlur(
+    mask: MPMask,
+    background: ImageSource,
+    blurAmount: number = 1.0
+  ): void {
+    this.applyBackgroundBlurWebGL(
+      mask.getAsWebGLTexture(),
+      background,
+      blurAmount
+    )
+  }
+
+  /**
+   * Frees all WebGL resources held by this class.
+   * @export
+   */
+  close(): void {
+    this.backgroundBlurShaderContext?.close()
+    this.backgroundBlurShaderContext = undefined
+  }
+}

src/frontend/src/features/rooms/livekit/components/blur/image_shader_context.ts

@@ -0,0 +1,324 @@
+/**
+ * Copyright 2023 The MediaPipe Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const VERTEX_SHADER = `#version 300 es
+  in vec2 aVertex;
+  in vec2 aTex;
+  out vec2 vTex;
+  void main(void) {
+    gl_Position = vec4(aVertex, 0.0, 1.0);
+    vTex = aTex;
+  }`
+
+const FRAGMENT_SHADER = `#version 300 es
+  precision mediump float;
+  in vec2 vTex;
+  uniform sampler2D inputTexture;
+  out vec4 fragColor;
+  void main() {
+    fragColor = texture(inputTexture, vTex);
+  }
+ `
+
+/** Helper to assert that `value` is not null or undefined.  */
+export function assertExists<T>(value: T, msg: string): NonNullable<T> {
+  if (!value) {
+    throw new Error(`Unable to obtain required WebGL resource: ${msg}`)
+  }
+  return value
+}
+
+/**
+ * Utility class that encapsulates the buffers used by `MPImageShaderContext`.
+ * For internal use only.
+ */
+class MPImageShaderBuffers {
+  constructor(
+    private readonly gl: WebGL2RenderingContext,
+    private readonly vertexArrayObject: WebGLVertexArrayObject,
+    private readonly vertexBuffer: WebGLBuffer,
+    private readonly textureBuffer: WebGLBuffer
+  ) {}
+
+  bind() {
+    this.gl.bindVertexArray(this.vertexArrayObject)
+  }
+
+  unbind() {
+    this.gl.bindVertexArray(null)
+  }
+
+  close() {
+    this.gl.deleteVertexArray(this.vertexArrayObject)
+    this.gl.deleteBuffer(this.vertexBuffer)
+    this.gl.deleteBuffer(this.textureBuffer)
+  }
+}
+
+/**
+ * A class that encapsulates the shaders used by an MPImage. Can be re-used
+ * across MPImages that use the same WebGL2Rendering context.
+ *
+ * For internal use only.
+ */
+export class MPImageShaderContext {
+  protected gl?: WebGL2RenderingContext
+  private framebuffer?: WebGLFramebuffer
+  protected program?: WebGLProgram
+  private vertexShader?: WebGLShader
+  private fragmentShader?: WebGLShader
+  private aVertex?: GLint
+  private aTex?: GLint
+
+  /**
+   * The shader buffers used for passthrough renders that don't modify the
+   * input texture.
+   */
+  private shaderBuffersPassthrough?: MPImageShaderBuffers
+
+  /**
+   * The shader buffers used for passthrough renders that flip the input texture
+   * vertically before conversion to a different type. This is used to flip the
+   * texture to the expected orientation for drawing in the browser.
+   */
+  private shaderBuffersFlipVertically?: MPImageShaderBuffers
+
+  protected getFragmentShader(): string {
+    return FRAGMENT_SHADER
+  }
+
+  protected getVertexShader(): string {
+    return VERTEX_SHADER
+  }
+
+  private compileShader(source: string, type: number): WebGLShader {
+    const gl = this.gl!
+    const shader = assertExists(
+      gl.createShader(type),
+      'Failed to create WebGL shader'
+    )
+    gl.shaderSource(shader, source)
+    gl.compileShader(shader)
+    if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {
+      const info = gl.getShaderInfoLog(shader)
+      throw new Error(`Could not compile WebGL shader: ${info}`)
+    }
+    gl.attachShader(this.program!, shader)
+    return shader
+  }
+
+  protected setupShaders(): void {
+    const gl = this.gl!
+    this.program = assertExists(
+      gl.createProgram()!,
+      'Failed to create WebGL program'
+    )
+
+    this.vertexShader = this.compileShader(
+      this.getVertexShader(),
+      gl.VERTEX_SHADER
+    )
+    this.fragmentShader = this.compileShader(
+      this.getFragmentShader(),
+      gl.FRAGMENT_SHADER
+    )
+
+    gl.linkProgram(this.program)
+    const linked = gl.getProgramParameter(this.program, gl.LINK_STATUS)
+    if (!linked) {
+      const info = gl.getProgramInfoLog(this.program)
+      throw new Error(`Error during program linking: ${info}`)
+    }
+
+    this.aVertex = gl.getAttribLocation(this.program, 'aVertex')
+    this.aTex = gl.getAttribLocation(this.program, 'aTex')
+  }
+
+  protected setupTextures(): void {}
+
+  protected configureUniforms(): void {}
+
+  private createBuffers(flipVertically: boolean): MPImageShaderBuffers {
+    const gl = this.gl!
+    const vertexArrayObject = assertExists(
+      gl.createVertexArray(),
+      'Failed to create vertex array'
+    )
+    gl.bindVertexArray(vertexArrayObject)
+
+    const vertexBuffer = assertExists(
+      gl.createBuffer(),
+      'Failed to create buffer'
+    )
+    gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer)
+    gl.enableVertexAttribArray(this.aVertex!)
+    gl.vertexAttribPointer(this.aVertex!, 2, gl.FLOAT, false, 0, 0)
+    gl.bufferData(
+      gl.ARRAY_BUFFER,
+      new Float32Array([-1, -1, -1, 1, 1, 1, 1, -1]),
+      gl.STATIC_DRAW
+    )
+
+    const textureBuffer = assertExists(
+      gl.createBuffer(),
+      'Failed to create buffer'
+    )
+    gl.bindBuffer(gl.ARRAY_BUFFER, textureBuffer)
+    gl.enableVertexAttribArray(this.aTex!)
+    gl.vertexAttribPointer(this.aTex!, 2, gl.FLOAT, false, 0, 0)
+
+    const bufferData = flipVertically
+      ? [0, 1, 0, 0, 1, 0, 1, 1]
+      : [0, 0, 0, 1, 1, 1, 1, 0]
+    gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(bufferData), gl.STATIC_DRAW)
+
+    gl.bindBuffer(gl.ARRAY_BUFFER, null)
+    gl.bindVertexArray(null)
+
+    return new MPImageShaderBuffers(
+      gl,
+      vertexArrayObject,
+      vertexBuffer,
+      textureBuffer
+    )
+  }
+
+  private getShaderBuffers(flipVertically: boolean): MPImageShaderBuffers {
+    if (flipVertically) {
+      if (!this.shaderBuffersFlipVertically) {
+        this.shaderBuffersFlipVertically = this.createBuffers(
+          /* flipVertically= */ true
+        )
+      }
+      return this.shaderBuffersFlipVertically
+    } else {
+      if (!this.shaderBuffersPassthrough) {
+        this.shaderBuffersPassthrough = this.createBuffers(
+          /* flipVertically= */ false
+        )
+      }
+      return this.shaderBuffersPassthrough
+    }
+  }
+
+  private maybeInitGL(gl: WebGL2RenderingContext): void {
+    if (!this.gl) {
+      this.gl = gl
+    } else if (gl !== this.gl) {
+      throw new Error('Cannot change GL context once initialized')
+    }
+  }
+
+  /** Runs the callback using the shader. */
+  run<T>(
+    gl: WebGL2RenderingContext,
+    flipVertically: boolean,
+    callback: () => T
+  ): T {
+    this.maybeInitGL(gl)
+
+    if (!this.program) {
+      this.setupShaders()
+      this.setupTextures()
+    }
+
+    const shaderBuffers = this.getShaderBuffers(flipVertically)
+    gl.useProgram(this.program!)
+    shaderBuffers.bind()
+    this.configureUniforms()
+    const result = callback()
+    shaderBuffers.unbind()
+
+    return result
+  }
+
+  /**
+   * Creates and configures a texture.
+   *
+   * @param gl The rendering context.
+   * @param filter The setting to use for `gl.TEXTURE_MIN_FILTER` and
+   *     `gl.TEXTURE_MAG_FILTER`. Defaults to `gl.LINEAR`.
+   * @param wrapping The setting to use for `gl.TEXTURE_WRAP_S` and
+   *     `gl.TEXTURE_WRAP_T`. Defaults to `gl.CLAMP_TO_EDGE`.
+   */
+  createTexture(
+    gl: WebGL2RenderingContext,
+    filter?: GLenum,
+    wrapping?: GLenum
+  ): WebGLTexture {
+    this.maybeInitGL(gl)
+    const texture = assertExists(gl.createTexture(), 'Failed to create texture')
+    gl.bindTexture(gl.TEXTURE_2D, texture)
+    gl.texParameteri(
+      gl.TEXTURE_2D,
+      gl.TEXTURE_WRAP_S,
+      wrapping ?? gl.CLAMP_TO_EDGE
+    )
+    gl.texParameteri(
+      gl.TEXTURE_2D,
+      gl.TEXTURE_WRAP_T,
+      wrapping ?? gl.CLAMP_TO_EDGE
+    )
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, filter ?? gl.LINEAR)
+    gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, filter ?? gl.LINEAR)
+    gl.bindTexture(gl.TEXTURE_2D, null)
+    return texture
+  }
+
+  /**
+   * Binds a framebuffer to the canvas. If the framebuffer does not yet exist,
+   * creates it first. Binds the provided texture to the framebuffer.
+   */
+  bindFramebuffer(gl: WebGL2RenderingContext, texture: WebGLTexture): void {
+    this.maybeInitGL(gl)
+    if (!this.framebuffer) {
+      this.framebuffer = assertExists(
+        gl.createFramebuffer(),
+        'Failed to create framebuffe.'
+      )
+    }
+    gl.bindFramebuffer(gl.FRAMEBUFFER, this.framebuffer)
+    gl.framebufferTexture2D(
+      gl.FRAMEBUFFER,
+      gl.COLOR_ATTACHMENT0,
+      gl.TEXTURE_2D,
+      texture,
+      0
+    )
+  }
+
+  unbindFramebuffer(): void {
+    this.gl?.bindFramebuffer(this.gl.FRAMEBUFFER, null)
+  }
+
+  close() {
+    if (this.program) {
+      const gl = this.gl!
+      gl.deleteProgram(this.program)
+      gl.deleteShader(this.vertexShader!)
+      gl.deleteShader(this.fragmentShader!)
+    }
+    if (this.framebuffer) {
+      this.gl!.deleteFramebuffer(this.framebuffer)
+    }
+    if (this.shaderBuffersPassthrough) {
+      this.shaderBuffersPassthrough.close()
+    }
+    if (this.shaderBuffersFlipVertically) {
+      this.shaderBuffersFlipVertically.close()
+    }
+  }
+}

src/frontend/src/features/rooms/livekit/components/blur/index.ts

@@ -36,22 +36,25 @@ export class BackgroundProcessorFactory {
     type: ProcessorType,
     opts: BackgroundOptions
   ): BackgroundProcessorInterface | undefined {
-    if (type === ProcessorType.BLUR) {
-      if (ProcessorWrapper.isSupported) {
-        return new BackgroundBlurTrackProcessorJsWrapper(opts)
-      }
-      if (BackgroundCustomProcessor.isSupported) {
-        return new BackgroundCustomProcessor(opts)
-      }
-    } else if (type === ProcessorType.VIRTUAL) {
-      if (ProcessorWrapper.isSupported) {
-        return new BackgroundVirtualTrackProcessorJsWrapper(opts)
-      }
-      if (BackgroundCustomProcessor.isSupported) {
-        return new BackgroundCustomProcessor(opts)
-      }
-    }
-    return undefined
+    return new BackgroundCustomProcessor(opts)
+
+    // if (type === ProcessorType.BLUR) {
+    //   if (ProcessorWrapper.isSupported) {
+    //
+    //   }
+    //   if (BackgroundCustomProcessor.isSupported) {
+    //     return new BackgroundCustomProcessor(opts)
+    //   }
+    // } else if (type === ProcessorType.VIRTUAL) {
+    //   if (ProcessorWrapper.isSupported) {
+    //     return new BackgroundVirtualTrackProcessorJsWrapper(opts)
+    //   }
+    //   if (BackgroundCustomProcessor.isSupported) {
+    //     return new BackgroundCustomProcessor(opts)
+    //   }
+    // }
+
+    // return undefined
   }
 
   static deserializeProcessor(data?: ProcessorSerialized) {

src/frontend/src/features/rooms/livekit/components/blur/wip.ts

@@ -0,0 +1,217 @@
+/**
+ * Copyright 2023 The MediaPipe Authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MPImageShaderContext, assertExists } from './image_shader_context'
+
+import { ImageSource } from '@mediapipe/tasks-vision'
+
+/**
+ * A fragment shader that maps categories to colors based on a background
+ * texture, a mask texture and a 256x1 "color mapping texture" that contains one
+ * color for each pixel.
+ */
+const FRAGMENT_SHADER = `
+  precision mediump float;
+  uniform sampler2D backgroundTexture;
+  uniform sampler2D maskTexture;
+  uniform sampler2D colorMappingTexture;
+  varying vec2 vTex;
+  void main() {
+    vec4 backgroundColor = texture2D(backgroundTexture, vTex);
+    float category = texture2D(maskTexture, vTex).r;
+    vec4 categoryColor = texture2D(colorMappingTexture, vec2(category, 0.0));
+    gl_FragColor = mix(backgroundColor, categoryColor, categoryColor.a);
+  }
+ `
+
+/**
+ * A four channel color with values for red, green, blue and alpha
+ * respectively.
+ */
+export type RGBAColor = [number, number, number, number] | number[]
+
+/**
+ * A category to color mapping that uses either a map or an array to assign
+ * category indexes to RGBA colors.
+ */
+export type CategoryToColorMap = Map<number, RGBAColor> | RGBAColor[]
+
+/** Checks CategoryToColorMap maps for deep equality. */
+function isEqualColorMap(
+  a: CategoryToColorMap,
+  b: CategoryToColorMap
+): boolean {
+  if (a !== b) {
+    return false
+  }
+
+  const aEntries = a.entries()
+  const bEntries = b.entries()
+  for (const [aKey, aValue] of aEntries) {
+    const bNext = bEntries.next()
+    if (bNext.done) {
+      return false
+    }
+
+    const [bKey, bValue] = bNext.value
+    if (aKey !== bKey) {
+      return false
+    }
+
+    if (
+      aValue[0] !== bValue[0] ||
+      aValue[1] !== bValue[1] ||
+      aValue[2] !== bValue[2] ||
+      aValue[3] !== bValue[3]
+    ) {
+      return false
+    }
+  }
+  return !!bEntries.next().done
+}
+
+/** A drawing util class for category masks. */
+export class CategoryMaskShaderContext extends MPImageShaderContext {
+  backgroundTexture?: WebGLTexture
+  colorMappingTexture?: WebGLTexture
+  colorMappingTextureUniform?: WebGLUniformLocation
+  backgroundTextureUniform?: WebGLUniformLocation
+  maskTextureUniform?: WebGLUniformLocation
+  currentColorMap?: CategoryToColorMap
+
+  bindAndUploadTextures(
+    categoryMask: WebGLTexture,
+    background: ImageSource,
+    colorMap: Map<number, number[]> | number[][]
+  ) {
+    const gl = this.gl!
+
+    // Bind category mask
+    gl.activeTexture(gl.TEXTURE0)
+    gl.bindTexture(gl.TEXTURE_2D, categoryMask)
+
+    // TODO: We should avoid uploading textures from CPU to GPU
+    // if the textures haven't changed. This can lead to drastic performance
+    // slowdowns (~50ms per frame). Users can reduce the penalty by passing a
+    // canvas object instead of ImageData/HTMLImageElement.
+    gl.activeTexture(gl.TEXTURE1)
+    gl.bindTexture(gl.TEXTURE_2D, this.backgroundTexture!)
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      background
+    )
+
+    // Bind color mapping texture if changed.
+    if (
+      !this.currentColorMap ||
+      !isEqualColorMap(this.currentColorMap, colorMap)
+    ) {
+      this.currentColorMap = colorMap
+
+      const pixels = new Array(256 * 4).fill(0)
+      colorMap.forEach((rgba, index) => {
+        if (rgba.length !== 4) {
+          throw new Error(
+            `Color at index ${index} is not a four-channel value.`
+          )
+        }
+        pixels[index * 4] = rgba[0]
+        pixels[index * 4 + 1] = rgba[1]
+        pixels[index * 4 + 2] = rgba[2]
+        pixels[index * 4 + 3] = rgba[3]
+      })
+      gl.activeTexture(gl.TEXTURE2)
+      gl.bindTexture(gl.TEXTURE_2D, this.colorMappingTexture!)
+      gl.texImage2D(
+        gl.TEXTURE_2D,
+        0,
+        gl.RGBA,
+        256,
+        1,
+        0,
+        gl.RGBA,
+        gl.UNSIGNED_BYTE,
+        new Uint8Array(pixels)
+      )
+    } else {
+      gl.activeTexture(gl.TEXTURE2)
+      gl.bindTexture(gl.TEXTURE_2D, this.colorMappingTexture!)
+    }
+  }
+
+  unbindTextures() {
+    const gl = this.gl!
+    gl.activeTexture(gl.TEXTURE0)
+    gl.bindTexture(gl.TEXTURE_2D, null)
+    gl.activeTexture(gl.TEXTURE1)
+    gl.bindTexture(gl.TEXTURE_2D, null)
+    gl.activeTexture(gl.TEXTURE2)
+    gl.bindTexture(gl.TEXTURE_2D, null)
+  }
+
+  protected override getFragmentShader(): string {
+    return FRAGMENT_SHADER
+  }
+
+  protected override setupTextures(): void {
+    const gl = this.gl!
+    gl.activeTexture(gl.TEXTURE1)
+    this.backgroundTexture = this.createTexture(gl, gl.LINEAR)
+    // Use `gl.NEAREST` to prevent interpolating values in our category to
+    // color map.
+    gl.activeTexture(gl.TEXTURE2)
+    this.colorMappingTexture = this.createTexture(gl, gl.NEAREST)
+  }
+
+  protected override setupShaders(): void {
+    super.setupShaders()
+    const gl = this.gl!
+    this.backgroundTextureUniform = assertExists(
+      gl.getUniformLocation(this.program!, 'backgroundTexture'),
+      'Uniform location'
+    )
+    this.colorMappingTextureUniform = assertExists(
+      gl.getUniformLocation(this.program!, 'colorMappingTexture'),
+      'Uniform location'
+    )
+    this.maskTextureUniform = assertExists(
+      gl.getUniformLocation(this.program!, 'maskTexture'),
+      'Uniform location'
+    )
+  }
+
+  protected override configureUniforms(): void {
+    super.configureUniforms()
+    const gl = this.gl!
+    gl.uniform1i(this.maskTextureUniform!, 0)
+    gl.uniform1i(this.backgroundTextureUniform!, 1)
+    gl.uniform1i(this.colorMappingTextureUniform!, 2)
+  }
+
+  override close(): void {
+    if (this.backgroundTexture) {
+      this.gl!.deleteTexture(this.backgroundTexture)
+    }
+    if (this.colorMappingTexture) {
+      this.gl!.deleteTexture(this.colorMappingTexture)
+    }
+    super.close()
+  }
+}

src/frontend/src/features/rooms/livekit/components/blur/wip2.ts

@@ -0,0 +1,202 @@
+/**
+ * Background Blur Shader Implementation
+ *
+ * Implements a high-quality background blur while preserving
+ * foreground based on a mask. Uses a multi-pass blur approach
+ * for a smoother, more professional-looking effect.
+ *
+ * This implementation INVERTS the mask to blur what's outside the mask.
+ */
+
+import { MPImageShaderContext, assertExists } from './image_shader_context'
+import { ImageSource } from '@mediapipe/tasks-vision'
+
+/**
+ * Two-pass blur with smooth transition between foreground and background.
+ * Uses a larger kernel and multiple passes for a higher quality blur.
+ *
+ * This version inverts the mask interpretation - it blurs what's OUTSIDE the mask.
+ */
+
+const FRAGMENT_SHADER = `#version 300 es
+  precision mediump float;
+  
+  uniform sampler2D backgroundTexture;
+  uniform sampler2D maskTexture;
+  uniform vec2 texelSize;
+  uniform float blurAmount;
+  
+  in vec2 vTex;
+  out vec4 fragColor;
+
+  // Smoothing function for mask transitions
+  vec4 maskSmoothing() {
+    // Sample the mask at the current texel
+    float maskValue = texture(maskTexture, vTex).r;
+    
+    // Apply smoothing to reduce aliasing at mask edges
+    float smoothedMask = smoothstep(0.2, 0.8, maskValue);
+    
+    return vec4(smoothedMask);
+  }
+  
+  // Compute Gaussian weight based on distance and blur amount
+  float gaussianWeight(float x, float y, float sigma) {
+    float sigmaSq = sigma * sigma;
+    float distanceSq = x * x + y * y;
+    
+    // Gaussian function: (1/(2πσ²)) * e^(-((x²+y²)/(2σ²)))
+    return exp(-distanceSq / (2.0 * sigmaSq));
+  }
+
+  void main() {
+    vec4 centerColor = texture(backgroundTexture, vTex);
+    
+    // Apply bilateral filtering on the mask before using it
+    vec4 personMask = maskSmoothing();
+    
+    // Dynamic 5x5 Gaussian blur
+    vec4 blurredColor = vec4(0.0);
+    float totalWeight = 0.0;
+    
+    // Compute effective sigma from blur amount (adjust as needed)
+    float sigma = max(0.1, blurAmount) + 50.0;
+    
+    // Apply 5x5 kernel with dynamic weights
+    for (int y = -10; y <= 10; y++) {
+      for (int x = -10; x <= 10; x++) {
+        // Calculate Gaussian weight for this offset
+        float weight = gaussianWeight(float(x), float(y), sigma);
+        totalWeight += weight;
+        
+        vec2 offset = vec2(float(x), float(y)) * texelSize;
+        vec2 sampleCoord = vTex + offset;
+        blurredColor += texture(backgroundTexture, sampleCoord) * weight;
+      }
+    }
+    
+    // Normalize by total weight
+    blurredColor /= totalWeight;
+    
+    // Blend original color with blurred background based on mask
+    fragColor = mix(centerColor, blurredColor, personMask.r);
+  }
+`;
+
+
+
+/** A drawing util class for high-quality background blur. */
+export class BackgroundBlurShaderContext extends MPImageShaderContext {
+  backgroundTexture?: WebGLTexture
+  maskTextureUniform?: WebGLUniformLocation
+  backgroundTextureUniform?: WebGLUniformLocation
+  texelSizeUniform?: WebGLUniformLocation
+  blurAmountUniform?: WebGLUniformLocation
+
+  bindAndUploadTextures(
+    foregroundMask: WebGLTexture,
+    background: ImageSource,
+    blurAmount: number = 1.0
+  ) {
+    const gl = this.gl!
+
+    // Bind foreground mask
+    gl.activeTexture(gl.TEXTURE0)
+    gl.bindTexture(gl.TEXTURE_2D, foregroundMask)
+
+    // Bind background texture
+    gl.activeTexture(gl.TEXTURE1)
+    gl.bindTexture(gl.TEXTURE_2D, this.backgroundTexture!)
+    gl.texImage2D(
+      gl.TEXTURE_2D,
+      0,
+      gl.RGBA,
+      gl.RGBA,
+      gl.UNSIGNED_BYTE,
+      background
+    )
+
+    // Set texel size (for proper blur scaling)
+    let width = 1;
+    let height = 1;
+
+    if (background instanceof ImageData) {
+      width = background.width;
+      height = background.height;
+    } else if (background instanceof HTMLCanvasElement) {
+      width = background.width;
+      height = background.height;
+    } else if (background instanceof HTMLImageElement) {
+      width = background.naturalWidth;
+      height = background.naturalHeight;
+    } else if (background instanceof HTMLVideoElement) {
+      width = background.videoWidth;
+      height = background.videoHeight;
+    }
+
+    // Ensure we have valid dimensions to prevent division by zero
+    width = Math.max(width, 1);
+    height = Math.max(height, 1);
+
+    gl.uniform2f(this.texelSizeUniform!, 1.0 / width, 1.0 / height)
+
+    // Set blur amount
+    if (this.blurAmountUniform) {
+      gl.uniform1f(this.blurAmountUniform, blurAmount)
+    }
+  }
+
+  unbindTextures() {
+    const gl = this.gl!
+    gl.activeTexture(gl.TEXTURE0)
+    gl.bindTexture(gl.TEXTURE_2D, null)
+    gl.activeTexture(gl.TEXTURE1)
+    gl.bindTexture(gl.TEXTURE_2D, null)
+  }
+
+  protected override getFragmentShader(): string {
+    return FRAGMENT_SHADER
+  }
+
+  protected override setupTextures(): void {
+    const gl = this.gl!
+    gl.activeTexture(gl.TEXTURE1)
+    this.backgroundTexture = this.createTexture(gl, gl.LINEAR)
+  }
+
+  protected override setupShaders(): void {
+    super.setupShaders()
+    const gl = this.gl!
+
+    this.backgroundTextureUniform = assertExists(
+      gl.getUniformLocation(this.program!, 'backgroundTexture'),
+      'Uniform location'
+    )
+    this.maskTextureUniform = assertExists(
+      gl.getUniformLocation(this.program!, 'maskTexture'),
+      'Uniform location'
+    )
+    this.texelSizeUniform = assertExists(
+      gl.getUniformLocation(this.program!, 'texelSize'),
+      'Uniform location'
+    )
+    this.blurAmountUniform = assertExists(
+      gl.getUniformLocation(this.program!, 'blurAmount'),
+      'Uniform location'
+    )
+  }
+
+  protected override configureUniforms(): void {
+    super.configureUniforms()
+    const gl = this.gl!
+    gl.uniform1i(this.maskTextureUniform!, 0)
+    gl.uniform1i(this.backgroundTextureUniform!, 1)
+  }
+
+  override close(): void {
+    if (this.backgroundTexture) {
+      this.gl!.deleteTexture(this.backgroundTexture)
+    }
+    super.close()
+  }
+}

src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx

@@ -170,16 +170,23 @@ export const EffectsConfiguration = ({
         })}
       >
         {videoTrack && !videoTrack.isMuted ? (
-          <video
-            ref={videoRef}
-            width="100%"
-            muted
-            style={{
-              transform: 'rotateY(180deg)',
-              minHeight: '175px',
-              borderRadius: '8px',
-            }}
-          />
+          <div
+            className={css({
+              position: 'relative',
+            })}
+            id="wip"
+          >
+            <video
+              ref={videoRef}
+              width="100%"
+              muted
+              style={{
+                transform: 'rotateY(180deg)',
+                minHeight: '175px',
+                borderRadius: '8px',
+              }}
+            />
+          </div>
         ) : (
           <div
             style={{