wip blur broken
This commit is contained in:
lebaudantoine
+7
-6
@@ -18,6 +18,7 @@ import {
|
||||
} from '.'
|
||||
|
||||
import { DrawingUtils } from './DrawingUtils'
|
||||
import { DrawingUtils2 } from '@/features/rooms/livekit/components/blur/DrawingUtils2.ts'
|
||||
|
||||
const PROCESSING_WIDTH = 256
|
||||
const PROCESSING_HEIGHT = 144
|
||||
@@ -166,8 +167,8 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
||||
this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {
|
||||
baseOptions: {
|
||||
modelAssetPath:
|
||||
// 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite',
|
||||
'https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/1/deeplab_v3.tflite',
|
||||
'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite',
|
||||
// 'https://storage.googleapis.com/mediapipe-models/image_segmenter/deeplab_v3/float32/1/deeplab_v3.tflite',
|
||||
delegate: 'GPU', // Use CPU for Firefox.
|
||||
},
|
||||
runningMode: 'VIDEO',
|
||||
@@ -202,10 +203,10 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
||||
|
||||
const legendColor = [[255, 197, 0, 255]]
|
||||
|
||||
this.drawingUtils.drawCategoryMask(
|
||||
this.drawingUtils.applyBackgroundBlur(
|
||||
this.imageSegmenterResult.categoryMask,
|
||||
legendColor, // Vivid Yellow
|
||||
this.videoElement
|
||||
this.videoElement,
|
||||
3.0
|
||||
)
|
||||
}
|
||||
|
||||
@@ -247,7 +248,7 @@ export class BackgroundCustomProcessor implements BackgroundProcessorInterface {
|
||||
PROCESSING_HEIGHT
|
||||
)
|
||||
}
|
||||
this.drawingUtils = new DrawingUtils(this.workCanvas.getContext('webgl2'))
|
||||
this.drawingUtils = new DrawingUtils2(this.workCanvas.getContext('webgl2'))
|
||||
}
|
||||
|
||||
_wip(id: string, width: number, height: number) {
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { MPMask } from '@mediapipe/tasks-vision'
|
||||
import { BackgroundBlurShaderContext } from './wip2.ts'
|
||||
|
||||
// Type definitions
|
||||
type ImageSource =
|
||||
| HTMLCanvasElement
|
||||
| HTMLImageElement
|
||||
| HTMLVideoElement
|
||||
| ImageData
|
||||
|
||||
export class DrawingUtils2 {
|
||||
private backgroundBlurShaderContext?: BackgroundBlurShaderContext
|
||||
private readonly contextWebGL?: WebGL2RenderingContext
|
||||
|
||||
/**
|
||||
* Creates a new DrawingUtils class.
|
||||
*
|
||||
* @param gpuContext The WebGL canvas rendering context to render into. If
|
||||
* your Task is using a GPU delegate, the context must be obtained from
|
||||
* its canvas (provided via `setOptions({ canvas: .. })`).
|
||||
*/
|
||||
constructor(gpuContext: WebGL2RenderingContext)
|
||||
/**
|
||||
* Creates a new DrawingUtils class.
|
||||
*
|
||||
* @param cpuContext The 2D canvas rendering context to render into. If
|
||||
* you are rendering GPU data you must also provide `gpuContext` to allow
|
||||
* for data conversion.
|
||||
* @param gpuContext A WebGL canvas that is used for GPU rendering and for
|
||||
* converting GPU to CPU data. If your Task is using a GPU delegate, the
|
||||
* context must be obtained from its canvas (provided via
|
||||
* `setOptions({ canvas: .. })`).
|
||||
*/
|
||||
constructor(
|
||||
cpuContext: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
|
||||
gpuContext?: WebGL2RenderingContext
|
||||
)
|
||||
constructor(
|
||||
cpuOrGpuContext:
|
||||
| CanvasRenderingContext2D
|
||||
| OffscreenCanvasRenderingContext2D
|
||||
| WebGL2RenderingContext,
|
||||
gpuContext?: WebGL2RenderingContext
|
||||
) {
|
||||
if (
|
||||
(typeof CanvasRenderingContext2D !== 'undefined' &&
|
||||
cpuOrGpuContext instanceof CanvasRenderingContext2D) ||
|
||||
cpuOrGpuContext instanceof OffscreenCanvasRenderingContext2D
|
||||
) {
|
||||
this.contextWebGL = gpuContext
|
||||
} else {
|
||||
// If the first condition is false, then the first argument must be a
|
||||
// WebGL2RenderingContext
|
||||
this.contextWebGL = cpuOrGpuContext as WebGL2RenderingContext
|
||||
}
|
||||
}
|
||||
|
||||
private getWebGLRenderingContext(): WebGL2RenderingContext {
|
||||
if (!this.contextWebGL) {
|
||||
throw new Error(
|
||||
'GPU rendering requested but WebGL2RenderingContext not provided.'
|
||||
)
|
||||
}
|
||||
return this.contextWebGL
|
||||
}
|
||||
|
||||
private getBackgroundBlurShaderContext(): BackgroundBlurShaderContext {
|
||||
if (!this.backgroundBlurShaderContext) {
|
||||
this.backgroundBlurShaderContext = new BackgroundBlurShaderContext()
|
||||
}
|
||||
return this.backgroundBlurShaderContext
|
||||
}
|
||||
|
||||
/** Applies background blur effect using WebGL2. */
|
||||
private applyBackgroundBlurWebGL(
|
||||
maskTexture: WebGLTexture,
|
||||
background: ImageSource,
|
||||
blurAmount: number = 1.0
|
||||
): void {
|
||||
const shaderContext = this.getBackgroundBlurShaderContext()
|
||||
const gl = this.getWebGLRenderingContext()
|
||||
|
||||
shaderContext.run(gl, /* flipTexturesVertically= */ true, () => {
|
||||
shaderContext.bindAndUploadTextures(maskTexture, background, blurAmount)
|
||||
gl.clearColor(0, 0, 0, 0)
|
||||
gl.clear(gl.COLOR_BUFFER_BIT)
|
||||
gl.drawArrays(gl.TRIANGLE_FAN, 0, 4)
|
||||
shaderContext.unbindTextures()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a background blur effect using the mask to preserve the foreground.
|
||||
*
|
||||
* @export
|
||||
* @param mask A binary mask that was returned from a segmentation task (1 for foreground, 0 for background).
|
||||
* @param background The input image to blur the background of.
|
||||
* @param blurAmount Optional blur amount parameter (defaults to 1.0, higher values create stronger blur).
|
||||
*/
|
||||
applyBackgroundBlur(
|
||||
mask: MPMask,
|
||||
background: ImageSource,
|
||||
blurAmount: number = 1.0
|
||||
): void {
|
||||
this.applyBackgroundBlurWebGL(
|
||||
mask.getAsWebGLTexture(),
|
||||
background,
|
||||
blurAmount
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Frees all WebGL resources held by this class.
|
||||
* @export
|
||||
*/
|
||||
close(): void {
|
||||
this.backgroundBlurShaderContext?.close()
|
||||
this.backgroundBlurShaderContext = undefined
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
/**
|
||||
* Background Blur Shader Implementation
|
||||
*
|
||||
* Implements a high-quality background blur while preserving
|
||||
* foreground based on a mask. Uses a multi-pass blur approach
|
||||
* for a smoother, more professional-looking effect.
|
||||
*
|
||||
* This implementation INVERTS the mask to blur what's outside the mask.
|
||||
*/
|
||||
|
||||
import { MPImageShaderContext, assertExists } from './image_shader_context'
|
||||
import { ImageSource } from '@mediapipe/tasks-vision'
|
||||
|
||||
/**
|
||||
* Two-pass blur with smooth transition between foreground and background.
|
||||
* Uses a larger kernel and multiple passes for a higher quality blur.
|
||||
*
|
||||
* This version inverts the mask interpretation - it blurs what's OUTSIDE the mask.
|
||||
*/
|
||||
const FRAGMENT_SHADER = `
|
||||
precision mediump float;
|
||||
|
||||
uniform sampler2D backgroundTexture;
|
||||
uniform sampler2D maskTexture;
|
||||
uniform vec2 texelSize;
|
||||
uniform float blurAmount;
|
||||
|
||||
varying vec2 vTex;
|
||||
|
||||
// Helper function for multi-pass blur
|
||||
vec4 multiPassBlur(sampler2D image, vec2 uv, vec2 pixelSize, float strength) {
|
||||
// Use a larger sampling area for higher quality blur
|
||||
vec4 accumulator = vec4(0.0);
|
||||
float weightSum = 0.0;
|
||||
float kernel[5];
|
||||
|
||||
// Define our approximated Gaussian weights
|
||||
kernel[0] = 0.227027;
|
||||
kernel[1] = 0.1945946;
|
||||
kernel[2] = 0.1216216;
|
||||
kernel[3] = 0.054054;
|
||||
kernel[4] = 0.016216;
|
||||
|
||||
// First horizontal pass
|
||||
// accumulator += texture2D(image, uv) * kernel[0];
|
||||
// weightSum += kernel[0];
|
||||
|
||||
for (int i = 0; i < 5; i++) {
|
||||
float weight = kernel[i];
|
||||
float offset = float(i) * pixelSize.x * strength;
|
||||
accumulator += texture2D(image, uv + vec2(offset, 0.0)) * weight;
|
||||
accumulator += texture2D(image, uv - vec2(offset, 0.0)) * weight;
|
||||
weightSum += weight * 2.0;
|
||||
}
|
||||
|
||||
vec4 horizontalPass = accumulator / weightSum;
|
||||
|
||||
// Reset for vertical pass
|
||||
// accumulator = vec4(0.0);
|
||||
// weightSum = 0.0;
|
||||
|
||||
// // Second vertical pass
|
||||
// accumulator += horizontalPass * kernel[0];
|
||||
// weightSum += kernel[0];
|
||||
//
|
||||
// for (int i = 1; i < 5; i++) {
|
||||
// float weight = kernel[i];
|
||||
// float offset = float(i) * pixelSize.y * strength;
|
||||
//
|
||||
// accumulator += horizontalPass * weight;
|
||||
// weightSum += weight;
|
||||
//
|
||||
// // Sample vertical offsets
|
||||
// vec2 offsetCoord;
|
||||
//
|
||||
// offsetCoord = uv + vec2(0.0, offset);
|
||||
// accumulator += texture2D(image, offsetCoord) * weight;
|
||||
//
|
||||
// offsetCoord = uv - vec2(0.0, offset);
|
||||
// accumulator += texture2D(image, offsetCoord) * weight;
|
||||
//
|
||||
// weightSum += weight * 2.0;
|
||||
// }
|
||||
|
||||
return accumulator / weightSum;
|
||||
}
|
||||
|
||||
// Create a smooth edge for the mask to prevent harsh transitions
|
||||
float smoothMask(float maskValue) {
|
||||
// Add a small amount of feathering to the mask edge
|
||||
float featherAmount = 0.02;
|
||||
return smoothstep(0.0, featherAmount, maskValue);
|
||||
}
|
||||
|
||||
void main() {
|
||||
// Get mask value and INVERT IT
|
||||
// Now: 0.0 for foreground (inside mask), 1.0 for background (outside mask)
|
||||
float rawMaskValue = texture2D(maskTexture, vTex).r;
|
||||
float maskValue = smoothMask(rawMaskValue);
|
||||
|
||||
// Original color (inside the mask - the "subject")
|
||||
vec4 originalColor = texture2D(backgroundTexture, vTex);
|
||||
|
||||
// Apply blur to outside of mask - stronger blur for a more dramatic effect
|
||||
vec4 blurredColor = multiPassBlur(backgroundTexture, vTex, texelSize, blurAmount * 6.0);
|
||||
|
||||
// Mix between blurred and original based on inverted mask
|
||||
// Now where mask was 1.0 (inside), maskValue is 0.0 (keep original)
|
||||
// Where mask was 0.0 (outside), maskValue is 1.0 (apply blur)
|
||||
// gl_FragColor = mix(originalColor, blurredColor, maskValue);
|
||||
gl_FragColor = blurredColor;
|
||||
}
|
||||
`
|
||||
|
||||
/** A drawing util class for high-quality background blur. */
|
||||
export class BackgroundBlurShaderContext extends MPImageShaderContext {
|
||||
backgroundTexture?: WebGLTexture
|
||||
maskTextureUniform?: WebGLUniformLocation
|
||||
backgroundTextureUniform?: WebGLUniformLocation
|
||||
texelSizeUniform?: WebGLUniformLocation
|
||||
blurAmountUniform?: WebGLUniformLocation
|
||||
|
||||
bindAndUploadTextures(
|
||||
foregroundMask: WebGLTexture,
|
||||
background: ImageSource,
|
||||
blurAmount: number = 1.0
|
||||
) {
|
||||
const gl = this.gl!
|
||||
|
||||
// Bind foreground mask
|
||||
gl.activeTexture(gl.TEXTURE0)
|
||||
gl.bindTexture(gl.TEXTURE_2D, foregroundMask)
|
||||
|
||||
// Bind background texture
|
||||
gl.activeTexture(gl.TEXTURE1)
|
||||
gl.bindTexture(gl.TEXTURE_2D, this.backgroundTexture!)
|
||||
gl.texImage2D(
|
||||
gl.TEXTURE_2D,
|
||||
0,
|
||||
gl.RGBA,
|
||||
gl.RGBA,
|
||||
gl.UNSIGNED_BYTE,
|
||||
background
|
||||
)
|
||||
|
||||
// Set texel size (for proper blur scaling)
|
||||
const width =
|
||||
background instanceof ImageData ? background.width : background.width || 1
|
||||
const height =
|
||||
background instanceof ImageData
|
||||
? background.height
|
||||
: background.height || 1
|
||||
|
||||
gl.uniform2f(this.texelSizeUniform!, 1.0 / width, 1.0 / height)
|
||||
|
||||
// Set blur amount
|
||||
gl.uniform1f(this.blurAmountUniform!, blurAmount)
|
||||
}
|
||||
|
||||
unbindTextures() {
|
||||
const gl = this.gl!
|
||||
gl.activeTexture(gl.TEXTURE0)
|
||||
gl.bindTexture(gl.TEXTURE_2D, null)
|
||||
gl.activeTexture(gl.TEXTURE1)
|
||||
gl.bindTexture(gl.TEXTURE_2D, null)
|
||||
}
|
||||
|
||||
protected override getFragmentShader(): string {
|
||||
return FRAGMENT_SHADER
|
||||
}
|
||||
|
||||
protected override setupTextures(): void {
|
||||
const gl = this.gl!
|
||||
gl.activeTexture(gl.TEXTURE1)
|
||||
this.backgroundTexture = this.createTexture(gl, gl.LINEAR)
|
||||
}
|
||||
|
||||
protected override setupShaders(): void {
|
||||
super.setupShaders()
|
||||
const gl = this.gl!
|
||||
|
||||
this.backgroundTextureUniform = assertExists(
|
||||
gl.getUniformLocation(this.program!, 'backgroundTexture'),
|
||||
'Uniform location'
|
||||
)
|
||||
this.maskTextureUniform = assertExists(
|
||||
gl.getUniformLocation(this.program!, 'maskTexture'),
|
||||
'Uniform location'
|
||||
)
|
||||
this.texelSizeUniform = assertExists(
|
||||
gl.getUniformLocation(this.program!, 'texelSize'),
|
||||
'Uniform location'
|
||||
)
|
||||
this.blurAmountUniform = assertExists(
|
||||
gl.getUniformLocation(this.program!, 'blurAmount'),
|
||||
'Uniform location'
|
||||
)
|
||||
}
|
||||
|
||||
protected override configureUniforms(): void {
|
||||
super.configureUniforms()
|
||||
const gl = this.gl!
|
||||
gl.uniform1i(this.maskTextureUniform!, 0)
|
||||
gl.uniform1i(this.backgroundTextureUniform!, 1)
|
||||
}
|
||||
|
||||
override close(): void {
|
||||
if (this.backgroundTexture) {
|
||||
this.gl!.deleteTexture(this.backgroundTexture)
|
||||
}
|
||||
super.close()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user