From 96310bff75e8145efff3bbf215eddaa2fbb186e8 Mon Sep 17 00:00:00 2001 From: lebaudantoine Date: Tue, 11 Mar 2025 19:45:34 +0100 Subject: [PATCH] wip copy/pasted mediapipe --- .../blur/BackgroundCustomProcessor.ts | 162 +++------ .../livekit/components/blur/DrawingUtils.ts | 158 +++++++++ .../components/blur/image_shader_context.ts | 323 ++++++++++++++++++ .../rooms/livekit/components/blur/index.ts | 35 +- .../rooms/livekit/components/blur/wip.ts | 217 ++++++++++++ .../effects/EffectsConfiguration.tsx | 27 +- 6 files changed, 782 insertions(+), 140 deletions(-) create mode 100644 src/frontend/src/features/rooms/livekit/components/blur/DrawingUtils.ts create mode 100644 src/frontend/src/features/rooms/livekit/components/blur/image_shader_context.ts create mode 100644 src/frontend/src/features/rooms/livekit/components/blur/wip.ts diff --git a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts index 744a749f..313fc292 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/BackgroundCustomProcessor.ts @@ -17,6 +17,8 @@ import { ProcessorType, } from '.' +import { DrawingUtils } from './DrawingUtils' + const PROCESSING_WIDTH = 256 const PROCESSING_HEIGHT = 144 @@ -25,6 +27,8 @@ const BLUR_CANVAS_ID = 'background-blur-local' const DEFAULT_BLUR = '10' +export class Wip {} + /** * This implementation of video blurring is made to be run on CPU for browser that are * not compatible with track-processor-js. @@ -50,8 +54,11 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { imageSegmenterResult?: ImageSegmenterResult // Canvas used for resizing video source and projecting mask. - segmentationMaskCanvas?: HTMLCanvasElement - segmentationMaskCanvasCtx?: CanvasRenderingContext2D + workCanvas?: HTMLCanvasElement + // segmentationMaskCanvasCtx?: CanvasRenderingContext2D + drawingUtils: any + segmentationMaskCanvasCtx: any + segmentationMaskCanvas: any // Mask containg the inference result. segmentationMask?: ImageData @@ -90,10 +97,11 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { this._initVirtualBackgroundImage() this._createMainCanvas() - this._createMaskCanvas() + this._createWorkCanvas() const stream = this.outputCanvas!.captureStream() const tracks = stream.getVideoTracks() + if (tracks.length == 0) { throw new Error('No tracks found for processing') } @@ -158,39 +166,17 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, { baseOptions: { modelAssetPath: - 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite', - delegate: 'CPU', // Use CPU for Firefox. + // 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite', + 'https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/1/deeplab_v3.tflite', + delegate: 'GPU', // Use CPU for Firefox. }, runningMode: 'VIDEO', outputCategoryMask: true, outputConfidenceMasks: false, + canvas: this.workCanvas, }) } - /** - * Resize the source video to the processing resolution. - */ - async sizeSource() { - this.segmentationMaskCanvasCtx?.drawImage( - this.videoElement!, - 0, - 0, - this.videoElement!.videoWidth, - this.videoElement!.videoHeight, - 0, - 0, - PROCESSING_WIDTH, - PROCESSING_HEIGHT - ) - - this.sourceImageData = this.segmentationMaskCanvasCtx?.getImageData( - 0, - 0, - PROCESSING_WIDTH, - PROCESSING_WIDTH - ) - } - /** * Run the segmentation. */ @@ -198,7 +184,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { const startTimeMs = performance.now() return new Promise((resolve) => { this.imageSegmenter!.segmentForVideo( - this.sourceImageData!, + this.videoElement, startTimeMs, (result: ImageSegmenterResult) => { this.imageSegmenterResult = result @@ -212,92 +198,26 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { * TODO: future improvement with WebGL. */ async blur() { - const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array() - for (let i = 0; i < mask.length; ++i) { - this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i] - } + // console.log('$$ blur') - this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0) + const legendColor = [[255, 197, 0, 255]] - this.outputCanvasCtx!.globalCompositeOperation = 'copy' - this.outputCanvasCtx!.filter = 'blur(8px)' - - // Put opacity mask. - this.outputCanvasCtx!.drawImage( - this.segmentationMaskCanvas!, - 0, - 0, - PROCESSING_WIDTH, - PROCESSING_HEIGHT, - 0, - 0, - this.videoElement!.videoWidth, - this.videoElement!.videoHeight - ) - - // Draw clear body. - this.outputCanvasCtx!.globalCompositeOperation = 'source-in' - this.outputCanvasCtx!.filter = 'none' - this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) - - // Draw blurry background. - this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' - this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)` - this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) - } - - /** - * TODO: future improvement with WebGL. - */ - async drawVirtualBackground() { - const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array() - for (let i = 0; i < mask.length; ++i) { - this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i] - } - - this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0) - - this.outputCanvasCtx!.globalCompositeOperation = 'copy' - this.outputCanvasCtx!.filter = 'blur(8px)' - - // Put opacity mask. - this.outputCanvasCtx!.drawImage( - this.segmentationMaskCanvas!, - 0, - 0, - PROCESSING_WIDTH, - PROCESSING_HEIGHT, - 0, - 0, - this.videoElement!.videoWidth, - this.videoElement!.videoHeight - ) - - // Draw clear body. - this.outputCanvasCtx!.globalCompositeOperation = 'source-in' - this.outputCanvasCtx!.filter = 'none' - this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0) - - // Draw virtual background. - this.outputCanvasCtx!.globalCompositeOperation = 'destination-over' - this.outputCanvasCtx!.drawImage( - this.virtualBackgroundImage!, - 0, - 0, - this.outputCanvas!.width, - this.outputCanvas!.height + this.drawingUtils.drawCategoryMask( + this.imageSegmenterResult.categoryMask, + legendColor, // Vivid Yellow + this.videoElement ) } async process() { - await this.sizeSource() await this.segment() if (this.options.blurRadius) { await this.blur() - } else { - await this.drawVirtualBackground() } + // } else { + // await this.drawVirtualBackground() + // } this.timerWorker!.postMessage({ id: SET_TIMEOUT, timeMs: 1000 / 30, @@ -318,19 +238,33 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface { this.outputCanvasCtx = this.outputCanvas.getContext('2d')! } - _createMaskCanvas() { - this.segmentationMaskCanvas = document.querySelector( - `#${SEGMENTATION_MASK_CANVAS_ID}` - ) as HTMLCanvasElement - if (!this.segmentationMaskCanvas) { - this.segmentationMaskCanvas = this._createCanvas( - SEGMENTATION_MASK_CANVAS_ID, + _createWorkCanvas() { + this.workCanvas = document.querySelector(`#workcanvas`) + if (!this.workCanvas) { + this.workCanvas = this._wip( + 'workcanvas', PROCESSING_WIDTH, PROCESSING_HEIGHT ) } - this.segmentationMaskCanvasCtx = - this.segmentationMaskCanvas.getContext('2d')! + this.drawingUtils = new DrawingUtils(this.workCanvas.getContext('webgl2')) + } + + _wip(id: string, width: number, height: number) { + const wip = document.querySelector(`#wip`) + const element = document.createElement('canvas') + element.setAttribute('id', id) + element.setAttribute('width', '' + width) + element.setAttribute('height', '' + height) + element.style.position = 'absolute' + element.style.top = '0' + element.style.left = '0' + element.style.maxWidth = '611px' + element.style.maxHeight = '305px' + + wip.appendChild(element) + + return element } _createCanvas(id: string, width: number, height: number) { diff --git a/src/frontend/src/features/rooms/livekit/components/blur/DrawingUtils.ts b/src/frontend/src/features/rooms/livekit/components/blur/DrawingUtils.ts new file mode 100644 index 00000000..54bf4300 --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/components/blur/DrawingUtils.ts @@ -0,0 +1,158 @@ +import { MPMask } from '@mediapipe/tasks-vision' +import { CategoryMaskShaderContext } from './wip' + +// Type definitions +type RGBAColor = [number, number, number, number] +type ImageSource = + | HTMLCanvasElement + | HTMLImageElement + | HTMLVideoElement + | ImageData + +type CategoryToColorMap = Map | RGBAColor[] + +export class DrawingUtils { + private categoryMaskShaderContext?: CategoryMaskShaderContext + private readonly contextWebGL?: WebGL2RenderingContext + + /** + * Creates a new DrawingUtils class. + * + * @param gpuContext The WebGL canvas rendering context to render into. If + * your Task is using a GPU delegate, the context must be obtained from + * its canvas (provided via `setOptions({ canvas: .. })`). + */ + constructor(gpuContext: WebGL2RenderingContext) + /** + * Creates a new DrawingUtils class. + * + * @param cpuContext The 2D canvas rendering context to render into. If + * you are rendering GPU data you must also provide `gpuContext` to allow + * for data conversion. + * @param gpuContext A WebGL canvas that is used for GPU rendering and for + * converting GPU to CPU data. If your Task is using a GPU delegate, the + * context must be obtained from its canvas (provided via + * `setOptions({ canvas: .. })`). + */ + constructor( + cpuContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, + gpuContext?: WebGL2RenderingContext + ) + constructor( + cpuOrGpuGontext: + | CanvasRenderingContext2D + | OffscreenCanvasRenderingContext2D + | WebGL2RenderingContext, + gpuContext?: WebGL2RenderingContext + ) { + if ( + (typeof CanvasRenderingContext2D !== 'undefined' && + cpuOrGpuGontext instanceof CanvasRenderingContext2D) || + cpuOrGpuGontext instanceof OffscreenCanvasRenderingContext2D + ) { + // this.context2d = cpuOrGpuGontext + this.contextWebGL = gpuContext + } else { + // If the first `if` statement is false, then the first argument must be a + // WebGL2RenderingContext, since CanvasRenderingContext2D can't be passed + // as the first argument. However, typescript isn't smart enough to infer + // this so we cast. + this.contextWebGL = cpuOrGpuGontext as WebGL2RenderingContext + } + } + + private getWebGLRenderingContext(): WebGL2RenderingContext { + if (!this.contextWebGL) { + throw new Error( + 'GPU rendering requested but WebGL2RenderingContext not provided.' + ) + } + return this.contextWebGL + } + + private getCategoryMaskShaderContext(): CategoryMaskShaderContext { + if (!this.categoryMaskShaderContext) { + this.categoryMaskShaderContext = new CategoryMaskShaderContext() + } + return this.categoryMaskShaderContext + } + + /** Draws a category mask on a WebGL2RenderingContext2D. */ + private drawCategoryMaskWebGL( + categoryTexture: WebGLTexture, + background: RGBAColor | ImageSource, + categoryToColorMap: Map | RGBAColor[] + ): void { + const shaderContext = this.getCategoryMaskShaderContext() + const gl = this.getWebGLRenderingContext() + + const backgroundImage = Array.isArray(background) + ? new ImageData(new Uint8ClampedArray(background), 1, 1) + : background + + shaderContext.run(gl, /* flipTexturesVertically= */ true, () => { + shaderContext.bindAndUploadTextures( + categoryTexture, + backgroundImage, + categoryToColorMap + ) + gl.clearColor(0, 0, 0, 0) + gl.clear(gl.COLOR_BUFFER_BIT) + gl.drawArrays(gl.TRIANGLE_FAN, 0, 4) + shaderContext.unbindTextures() + }) + } + + /** + * Draws a category mask using the provided category-to-color mapping. + * + * @export + * @param mask A category mask that was returned from a segmentation task. + * @param categoryToColorMap A map that maps category indices to RGBA + * values. You must specify a map entry for each category. + * @param background A color or image to use as the background. Defaults to + * black. + */ + drawCategoryMask( + mask: MPMask, + categoryToColorMap: Map, + background?: RGBAColor | ImageSource + ): void + /** + * Draws a category mask using the provided color array. + * + * @export + * @param mask A category mask that was returned from a segmentation task. + * @param categoryToColorMap An array that maps indices to RGBA values. The + * array's indices must correspond to the category indices of the model + * and an entry must be provided for each category. + * @param background A color or image to use as the background. Defaults to + * black. + */ + drawCategoryMask( + mask: MPMask, + categoryToColorMap: RGBAColor[], + background?: RGBAColor | ImageSource + ): void + /** @export */ + drawCategoryMask( + mask: MPMask, + categoryToColorMap: CategoryToColorMap, + background: RGBAColor | ImageSource = [0, 0, 0, 255] + ): void { + this.drawCategoryMaskWebGL( + mask.getAsWebGLTexture(), + background, + categoryToColorMap + ) + } + + /** + * Frees all WebGL resources held by this class. + * @export + */ + close(): void { + this.categoryMaskShaderContext?.close() + this.categoryMaskShaderContext = undefined + } +} diff --git a/src/frontend/src/features/rooms/livekit/components/blur/image_shader_context.ts b/src/frontend/src/features/rooms/livekit/components/blur/image_shader_context.ts new file mode 100644 index 00000000..3106f454 --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/components/blur/image_shader_context.ts @@ -0,0 +1,323 @@ +/** + * Copyright 2023 The MediaPipe Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const VERTEX_SHADER = ` + attribute vec2 aVertex; + attribute vec2 aTex; + varying vec2 vTex; + void main(void) { + gl_Position = vec4(aVertex, 0.0, 1.0); + vTex = aTex; + }` + +const FRAGMENT_SHADER = ` + precision mediump float; + varying vec2 vTex; + uniform sampler2D inputTexture; + void main() { + gl_FragColor = texture2D(inputTexture, vTex); + } + ` + +/** Helper to assert that `value` is not null or undefined. */ +export function assertExists(value: T, msg: string): NonNullable { + if (!value) { + throw new Error(`Unable to obtain required WebGL resource: ${msg}`) + } + return value +} + +/** + * Utility class that encapsulates the buffers used by `MPImageShaderContext`. + * For internal use only. + */ +class MPImageShaderBuffers { + constructor( + private readonly gl: WebGL2RenderingContext, + private readonly vertexArrayObject: WebGLVertexArrayObject, + private readonly vertexBuffer: WebGLBuffer, + private readonly textureBuffer: WebGLBuffer + ) {} + + bind() { + this.gl.bindVertexArray(this.vertexArrayObject) + } + + unbind() { + this.gl.bindVertexArray(null) + } + + close() { + this.gl.deleteVertexArray(this.vertexArrayObject) + this.gl.deleteBuffer(this.vertexBuffer) + this.gl.deleteBuffer(this.textureBuffer) + } +} + +/** + * A class that encapsulates the shaders used by an MPImage. Can be re-used + * across MPImages that use the same WebGL2Rendering context. + * + * For internal use only. + */ +export class MPImageShaderContext { + protected gl?: WebGL2RenderingContext + private framebuffer?: WebGLFramebuffer + protected program?: WebGLProgram + private vertexShader?: WebGLShader + private fragmentShader?: WebGLShader + private aVertex?: GLint + private aTex?: GLint + + /** + * The shader buffers used for passthrough renders that don't modify the + * input texture. + */ + private shaderBuffersPassthrough?: MPImageShaderBuffers + + /** + * The shader buffers used for passthrough renders that flip the input texture + * vertically before conversion to a different type. This is used to flip the + * texture to the expected orientation for drawing in the browser. + */ + private shaderBuffersFlipVertically?: MPImageShaderBuffers + + protected getFragmentShader(): string { + return FRAGMENT_SHADER + } + + protected getVertexShader(): string { + return VERTEX_SHADER + } + + private compileShader(source: string, type: number): WebGLShader { + const gl = this.gl! + const shader = assertExists( + gl.createShader(type), + 'Failed to create WebGL shader' + ) + gl.shaderSource(shader, source) + gl.compileShader(shader) + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + const info = gl.getShaderInfoLog(shader) + throw new Error(`Could not compile WebGL shader: ${info}`) + } + gl.attachShader(this.program!, shader) + return shader + } + + protected setupShaders(): void { + const gl = this.gl! + this.program = assertExists( + gl.createProgram()!, + 'Failed to create WebGL program' + ) + + this.vertexShader = this.compileShader( + this.getVertexShader(), + gl.VERTEX_SHADER + ) + this.fragmentShader = this.compileShader( + this.getFragmentShader(), + gl.FRAGMENT_SHADER + ) + + gl.linkProgram(this.program) + const linked = gl.getProgramParameter(this.program, gl.LINK_STATUS) + if (!linked) { + const info = gl.getProgramInfoLog(this.program) + throw new Error(`Error during program linking: ${info}`) + } + + this.aVertex = gl.getAttribLocation(this.program, 'aVertex') + this.aTex = gl.getAttribLocation(this.program, 'aTex') + } + + protected setupTextures(): void {} + + protected configureUniforms(): void {} + + private createBuffers(flipVertically: boolean): MPImageShaderBuffers { + const gl = this.gl! + const vertexArrayObject = assertExists( + gl.createVertexArray(), + 'Failed to create vertex array' + ) + gl.bindVertexArray(vertexArrayObject) + + const vertexBuffer = assertExists( + gl.createBuffer(), + 'Failed to create buffer' + ) + gl.bindBuffer(gl.ARRAY_BUFFER, vertexBuffer) + gl.enableVertexAttribArray(this.aVertex!) + gl.vertexAttribPointer(this.aVertex!, 2, gl.FLOAT, false, 0, 0) + gl.bufferData( + gl.ARRAY_BUFFER, + new Float32Array([-1, -1, -1, 1, 1, 1, 1, -1]), + gl.STATIC_DRAW + ) + + const textureBuffer = assertExists( + gl.createBuffer(), + 'Failed to create buffer' + ) + gl.bindBuffer(gl.ARRAY_BUFFER, textureBuffer) + gl.enableVertexAttribArray(this.aTex!) + gl.vertexAttribPointer(this.aTex!, 2, gl.FLOAT, false, 0, 0) + + const bufferData = flipVertically + ? [0, 1, 0, 0, 1, 0, 1, 1] + : [0, 0, 0, 1, 1, 1, 1, 0] + gl.bufferData(gl.ARRAY_BUFFER, new Float32Array(bufferData), gl.STATIC_DRAW) + + gl.bindBuffer(gl.ARRAY_BUFFER, null) + gl.bindVertexArray(null) + + return new MPImageShaderBuffers( + gl, + vertexArrayObject, + vertexBuffer, + textureBuffer + ) + } + + private getShaderBuffers(flipVertically: boolean): MPImageShaderBuffers { + if (flipVertically) { + if (!this.shaderBuffersFlipVertically) { + this.shaderBuffersFlipVertically = this.createBuffers( + /* flipVertically= */ true + ) + } + return this.shaderBuffersFlipVertically + } else { + if (!this.shaderBuffersPassthrough) { + this.shaderBuffersPassthrough = this.createBuffers( + /* flipVertically= */ false + ) + } + return this.shaderBuffersPassthrough + } + } + + private maybeInitGL(gl: WebGL2RenderingContext): void { + if (!this.gl) { + this.gl = gl + } else if (gl !== this.gl) { + throw new Error('Cannot change GL context once initialized') + } + } + + /** Runs the callback using the shader. */ + run( + gl: WebGL2RenderingContext, + flipVertically: boolean, + callback: () => T + ): T { + this.maybeInitGL(gl) + + if (!this.program) { + this.setupShaders() + this.setupTextures() + } + + const shaderBuffers = this.getShaderBuffers(flipVertically) + gl.useProgram(this.program!) + shaderBuffers.bind() + this.configureUniforms() + const result = callback() + shaderBuffers.unbind() + + return result + } + + /** + * Creates and configures a texture. + * + * @param gl The rendering context. + * @param filter The setting to use for `gl.TEXTURE_MIN_FILTER` and + * `gl.TEXTURE_MAG_FILTER`. Defaults to `gl.LINEAR`. + * @param wrapping The setting to use for `gl.TEXTURE_WRAP_S` and + * `gl.TEXTURE_WRAP_T`. Defaults to `gl.CLAMP_TO_EDGE`. + */ + createTexture( + gl: WebGL2RenderingContext, + filter?: GLenum, + wrapping?: GLenum + ): WebGLTexture { + this.maybeInitGL(gl) + const texture = assertExists(gl.createTexture(), 'Failed to create texture') + gl.bindTexture(gl.TEXTURE_2D, texture) + gl.texParameteri( + gl.TEXTURE_2D, + gl.TEXTURE_WRAP_S, + wrapping ?? gl.CLAMP_TO_EDGE + ) + gl.texParameteri( + gl.TEXTURE_2D, + gl.TEXTURE_WRAP_T, + wrapping ?? gl.CLAMP_TO_EDGE + ) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, filter ?? gl.LINEAR) + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, filter ?? gl.LINEAR) + gl.bindTexture(gl.TEXTURE_2D, null) + return texture + } + + /** + * Binds a framebuffer to the canvas. If the framebuffer does not yet exist, + * creates it first. Binds the provided texture to the framebuffer. + */ + bindFramebuffer(gl: WebGL2RenderingContext, texture: WebGLTexture): void { + this.maybeInitGL(gl) + if (!this.framebuffer) { + this.framebuffer = assertExists( + gl.createFramebuffer(), + 'Failed to create framebuffe.' + ) + } + gl.bindFramebuffer(gl.FRAMEBUFFER, this.framebuffer) + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + gl.COLOR_ATTACHMENT0, + gl.TEXTURE_2D, + texture, + 0 + ) + } + + unbindFramebuffer(): void { + this.gl?.bindFramebuffer(this.gl.FRAMEBUFFER, null) + } + + close() { + if (this.program) { + const gl = this.gl! + gl.deleteProgram(this.program) + gl.deleteShader(this.vertexShader!) + gl.deleteShader(this.fragmentShader!) + } + if (this.framebuffer) { + this.gl!.deleteFramebuffer(this.framebuffer) + } + if (this.shaderBuffersPassthrough) { + this.shaderBuffersPassthrough.close() + } + if (this.shaderBuffersFlipVertically) { + this.shaderBuffersFlipVertically.close() + } + } +} diff --git a/src/frontend/src/features/rooms/livekit/components/blur/index.ts b/src/frontend/src/features/rooms/livekit/components/blur/index.ts index 893b9eb0..b606cf36 100644 --- a/src/frontend/src/features/rooms/livekit/components/blur/index.ts +++ b/src/frontend/src/features/rooms/livekit/components/blur/index.ts @@ -36,22 +36,25 @@ export class BackgroundProcessorFactory { type: ProcessorType, opts: BackgroundOptions ): BackgroundProcessorInterface | undefined { - if (type === ProcessorType.BLUR) { - if (ProcessorWrapper.isSupported) { - return new BackgroundBlurTrackProcessorJsWrapper(opts) - } - if (BackgroundCustomProcessor.isSupported) { - return new BackgroundCustomProcessor(opts) - } - } else if (type === ProcessorType.VIRTUAL) { - if (ProcessorWrapper.isSupported) { - return new BackgroundVirtualTrackProcessorJsWrapper(opts) - } - if (BackgroundCustomProcessor.isSupported) { - return new BackgroundCustomProcessor(opts) - } - } - return undefined + return new BackgroundCustomProcessor(opts) + + // if (type === ProcessorType.BLUR) { + // if (ProcessorWrapper.isSupported) { + // + // } + // if (BackgroundCustomProcessor.isSupported) { + // return new BackgroundCustomProcessor(opts) + // } + // } else if (type === ProcessorType.VIRTUAL) { + // if (ProcessorWrapper.isSupported) { + // return new BackgroundVirtualTrackProcessorJsWrapper(opts) + // } + // if (BackgroundCustomProcessor.isSupported) { + // return new BackgroundCustomProcessor(opts) + // } + // } + + // return undefined } static deserializeProcessor(data?: ProcessorSerialized) { diff --git a/src/frontend/src/features/rooms/livekit/components/blur/wip.ts b/src/frontend/src/features/rooms/livekit/components/blur/wip.ts new file mode 100644 index 00000000..7466ba8d --- /dev/null +++ b/src/frontend/src/features/rooms/livekit/components/blur/wip.ts @@ -0,0 +1,217 @@ +/** + * Copyright 2023 The MediaPipe Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { MPImageShaderContext, assertExists } from './image_shader_context' + +import { ImageSource } from '@mediapipe/tasks-vision' + +/** + * A fragment shader that maps categories to colors based on a background + * texture, a mask texture and a 256x1 "color mapping texture" that contains one + * color for each pixel. + */ +const FRAGMENT_SHADER = ` + precision mediump float; + uniform sampler2D backgroundTexture; + uniform sampler2D maskTexture; + uniform sampler2D colorMappingTexture; + varying vec2 vTex; + void main() { + vec4 backgroundColor = texture2D(backgroundTexture, vTex); + float category = texture2D(maskTexture, vTex).r; + vec4 categoryColor = texture2D(colorMappingTexture, vec2(category, 0.0)); + gl_FragColor = mix(backgroundColor, categoryColor, categoryColor.a); + } + ` + +/** + * A four channel color with values for red, green, blue and alpha + * respectively. + */ +export type RGBAColor = [number, number, number, number] | number[] + +/** + * A category to color mapping that uses either a map or an array to assign + * category indexes to RGBA colors. + */ +export type CategoryToColorMap = Map | RGBAColor[] + +/** Checks CategoryToColorMap maps for deep equality. */ +function isEqualColorMap( + a: CategoryToColorMap, + b: CategoryToColorMap +): boolean { + if (a !== b) { + return false + } + + const aEntries = a.entries() + const bEntries = b.entries() + for (const [aKey, aValue] of aEntries) { + const bNext = bEntries.next() + if (bNext.done) { + return false + } + + const [bKey, bValue] = bNext.value + if (aKey !== bKey) { + return false + } + + if ( + aValue[0] !== bValue[0] || + aValue[1] !== bValue[1] || + aValue[2] !== bValue[2] || + aValue[3] !== bValue[3] + ) { + return false + } + } + return !!bEntries.next().done +} + +/** A drawing util class for category masks. */ +export class CategoryMaskShaderContext extends MPImageShaderContext { + backgroundTexture?: WebGLTexture + colorMappingTexture?: WebGLTexture + colorMappingTextureUniform?: WebGLUniformLocation + backgroundTextureUniform?: WebGLUniformLocation + maskTextureUniform?: WebGLUniformLocation + currentColorMap?: CategoryToColorMap + + bindAndUploadTextures( + categoryMask: WebGLTexture, + background: ImageSource, + colorMap: Map | number[][] + ) { + const gl = this.gl! + + // Bind category mask + gl.activeTexture(gl.TEXTURE0) + gl.bindTexture(gl.TEXTURE_2D, categoryMask) + + // TODO: We should avoid uploading textures from CPU to GPU + // if the textures haven't changed. This can lead to drastic performance + // slowdowns (~50ms per frame). Users can reduce the penalty by passing a + // canvas object instead of ImageData/HTMLImageElement. + gl.activeTexture(gl.TEXTURE1) + gl.bindTexture(gl.TEXTURE_2D, this.backgroundTexture!) + gl.texImage2D( + gl.TEXTURE_2D, + 0, + gl.RGBA, + gl.RGBA, + gl.UNSIGNED_BYTE, + background + ) + + // Bind color mapping texture if changed. + if ( + !this.currentColorMap || + !isEqualColorMap(this.currentColorMap, colorMap) + ) { + this.currentColorMap = colorMap + + const pixels = new Array(256 * 4).fill(0) + colorMap.forEach((rgba, index) => { + if (rgba.length !== 4) { + throw new Error( + `Color at index ${index} is not a four-channel value.` + ) + } + pixels[index * 4] = rgba[0] + pixels[index * 4 + 1] = rgba[1] + pixels[index * 4 + 2] = rgba[2] + pixels[index * 4 + 3] = rgba[3] + }) + gl.activeTexture(gl.TEXTURE2) + gl.bindTexture(gl.TEXTURE_2D, this.colorMappingTexture!) + gl.texImage2D( + gl.TEXTURE_2D, + 0, + gl.RGBA, + 256, + 1, + 0, + gl.RGBA, + gl.UNSIGNED_BYTE, + new Uint8Array(pixels) + ) + } else { + gl.activeTexture(gl.TEXTURE2) + gl.bindTexture(gl.TEXTURE_2D, this.colorMappingTexture!) + } + } + + unbindTextures() { + const gl = this.gl! + gl.activeTexture(gl.TEXTURE0) + gl.bindTexture(gl.TEXTURE_2D, null) + gl.activeTexture(gl.TEXTURE1) + gl.bindTexture(gl.TEXTURE_2D, null) + gl.activeTexture(gl.TEXTURE2) + gl.bindTexture(gl.TEXTURE_2D, null) + } + + protected override getFragmentShader(): string { + return FRAGMENT_SHADER + } + + protected override setupTextures(): void { + const gl = this.gl! + gl.activeTexture(gl.TEXTURE1) + this.backgroundTexture = this.createTexture(gl, gl.LINEAR) + // Use `gl.NEAREST` to prevent interpolating values in our category to + // color map. + gl.activeTexture(gl.TEXTURE2) + this.colorMappingTexture = this.createTexture(gl, gl.NEAREST) + } + + protected override setupShaders(): void { + super.setupShaders() + const gl = this.gl! + this.backgroundTextureUniform = assertExists( + gl.getUniformLocation(this.program!, 'backgroundTexture'), + 'Uniform location' + ) + this.colorMappingTextureUniform = assertExists( + gl.getUniformLocation(this.program!, 'colorMappingTexture'), + 'Uniform location' + ) + this.maskTextureUniform = assertExists( + gl.getUniformLocation(this.program!, 'maskTexture'), + 'Uniform location' + ) + } + + protected override configureUniforms(): void { + super.configureUniforms() + const gl = this.gl! + gl.uniform1i(this.maskTextureUniform!, 0) + gl.uniform1i(this.backgroundTextureUniform!, 1) + gl.uniform1i(this.colorMappingTextureUniform!, 2) + } + + override close(): void { + if (this.backgroundTexture) { + this.gl!.deleteTexture(this.backgroundTexture) + } + if (this.colorMappingTexture) { + this.gl!.deleteTexture(this.colorMappingTexture) + } + super.close() + } +} diff --git a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx index b5765ded..4abc2675 100644 --- a/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx +++ b/src/frontend/src/features/rooms/livekit/components/effects/EffectsConfiguration.tsx @@ -170,16 +170,23 @@ export const EffectsConfiguration = ({ })} > {videoTrack && !videoTrack.isMuted ? ( -