Compare commits

..

2 Commits

Author SHA1 Message Date
Nathan Vasse 90b14955d8 (front) add posthog log for Firefox blurring
We want to be able to track device settings and versions when a user
enables this feature.
2025-01-14 17:41:49 +01:00
Nathan Vasse a5ea63606b (front) add blurring support on Firefox
We cannot use track-processor-js for Firefox because it uses various
approaches not compatible with Firefox. Firefox cannot run inference
on GPU, so we must use CPU, we could also not use MediaStreamTrackProcessor nor
MediaStreamTrackGenerator. So I had to make a new implementation from
the ground up, using canvas filters and CPU inferences.
2025-01-14 17:41:49 +01:00
23 changed files with 457 additions and 702 deletions
+4 -5
View File
@@ -130,18 +130,17 @@ class RoomSerializer(serializers.ModelSerializer):
del output["configuration"]
if role is not None or instance.is_public:
room_id = f"{instance.id!s}"
slug = f"{instance.id!s}"
username = request.query_params.get("username", None)
output["livekit"] = {
"url": settings.LIVEKIT_CONFIGURATION["url"],
"room": room_id,
"room": slug,
"token": utils.generate_token(
room=room_id, user=request.user, username=username
room=slug, user=request.user, username=username
),
"passphrase": utils.get_cached_passphrase(room_id)
}
output["is_administrable"] = is_admin
return output
-36
View File
@@ -47,8 +47,6 @@ from core.recording.worker.mediator import (
from . import permissions, serializers
from livekit import api as livekit_api
# pylint: disable=too-many-ancestors
logger = getLogger(__name__)
@@ -212,10 +210,6 @@ class RoomViewSet(
Allow unregistered rooms when activated.
For unregistered rooms we only return a null id and the livekit room and token.
"""
# todo - determine whether encryption is needed store a shared secret in memory or in redis
# todo - check if a secret already exists, else create one.
try:
instance = self.get_object()
except Http404:
@@ -349,36 +343,6 @@ class RoomViewSet(
{"message": f"Recording stopped for room {room.slug}."}
)
@decorators.action(
detail=False,
methods=["post"],
url_path="livekit-webhook",
permission_classes=[],
authentication_classes=[],
)
def handle_livekit_webhook(self, request, pk=None): # pylint: disable=unused-argument
"""Handle LiveKit webhook events."""
auth_token = request.headers.get("Authorization")
if not auth_token:
return drf_response.Response(
{"error": "Missing LiveKit authentication token"},
status=drf_status.HTTP_401_UNAUTHORIZED
)
token_verifier = livekit_api.TokenVerifier()
webhook_receiver = livekit_api.WebhookReceiver(token_verifier)
webhook_data = webhook_receiver.receive(request.body.decode("utf-8"), auth_token)
# Todo - livekit triggers a webhook for all events, see if we can restrict webhook to a limited number of events.
# Todo - handle Egress stopped / aborted events.
if webhook_data.event == "room_finished":
room_id = webhook_data.room.name
utils.clear_cache_passphrase(room_id)
return drf_response.Response({"message": f"Event processed"})
class ResourceAccessListModelMixin:
"""List mixin for resource access API."""
+1 -30
View File
@@ -1,7 +1,7 @@
"""Authentication Backends for the Meet core app."""
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
from django.core.exceptions import SuspiciousOperation
from django.utils.translation import gettext_lazy as _
import requests
@@ -10,11 +10,6 @@ from mozilla_django_oidc.auth import (
)
from core.models import User
from core.services.marketing_service import (
ContactCreationError,
ContactData,
get_marketing_service,
)
class OIDCAuthenticationBackend(MozillaOIDCAuthenticationBackend):
@@ -91,10 +86,6 @@ class OIDCAuthenticationBackend(MozillaOIDCAuthenticationBackend):
password="!", # noqa: S106
**claims,
)
if settings.SIGNUP_NEW_USER_TO_MARKETING_EMAIL:
self.signup_to_marketing_email(email)
elif not user:
return None
@@ -105,26 +96,6 @@ class OIDCAuthenticationBackend(MozillaOIDCAuthenticationBackend):
return user
@staticmethod
def signup_to_marketing_email(email):
"""Pragmatic approach to newsletter signup during authentication flow.
Details:
1. Uses a very short timeout (1s) to prevent blocking the auth process
2. Silently fails if the marketing service is down/slow to prioritize user experience
3. Trade-off: May miss some signups but ensures auth flow remains fast
Note: For a more robust solution, consider using Async task processing (Celery/Django-Q)
"""
try:
marketing_service = get_marketing_service()
contact_data = ContactData(
email=email, attributes={"VISIO_SOURCE": ["SIGNIN"]}
)
marketing_service.create_contact(contact_data, timeout=1)
except (ContactCreationError, ImproperlyConfigured, ImportError):
pass
def get_existing_user(self, sub, email):
"""Fetch existing user by sub or email."""
try:
@@ -1,134 +0,0 @@
"""Marketing service in charge of pushing data for marketing automation."""
import logging
from dataclasses import dataclass
from functools import lru_cache
from typing import Dict, List, Optional, Protocol
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.utils.module_loading import import_string
import brevo_python
logger = logging.getLogger(__name__)
class ContactCreationError(Exception):
"""Raised when the contact creation fails."""
@dataclass
class ContactData:
"""Contact data for marketing service integration."""
email: str
attributes: Optional[Dict[str, str]] = None
list_ids: Optional[List[int]] = None
update_enabled: bool = True
class MarketingServiceProtocol(Protocol):
"""Interface for marketing automation service integrations."""
def create_contact(
self, contact_data: ContactData, timeout: Optional[int] = None
) -> dict:
"""Create or update a contact.
Args:
contact_data: Contact information and attributes
timeout: API request timeout in seconds
Returns:
dict: Service response
Raises:
ContactCreationError: If contact creation fails
"""
class BrevoMarketingService:
"""Brevo marketing automation integration.
Handles:
- Contact management and segmentation
- Marketing campaigns and automation
- Email communications
Configuration via Django settings:
- BREVO_API_KEY: API authentication
- BREVO_API_CONTACT_LIST_IDS: Default contact lists
- BREVO_API_CONTACT_ATTRIBUTES: Default contact attributes
"""
def __init__(self):
"""Initialize Brevo (ex-sendinblue) marketing service."""
if not settings.BREVO_API_KEY:
raise ImproperlyConfigured("Brevo API key is required")
configuration = brevo_python.Configuration()
configuration.api_key["api-key"] = settings.BREVO_API_KEY
self._api_client = brevo_python.ApiClient(configuration)
def create_contact(self, contact_data: ContactData, timeout=None) -> dict:
"""Create or update a Brevo contact.
Args:
contact_data: Contact information and attributes
timeout: API request timeout in seconds
Returns:
dict: Brevo API response
Raises:
ContactCreationError: If contact creation fails
ImproperlyConfigured: If required settings are missing
Note:
Contact attributes must be pre-configured in Brevo.
Changes to attributes can impact existing workflows.
"""
if not settings.BREVO_API_CONTACT_LIST_IDS:
raise ImproperlyConfigured(
"Default Brevo List IDs must be configured in settings."
)
contact_api = brevo_python.ContactsApi(self._api_client)
attributes = {
**settings.BREVO_API_CONTACT_ATTRIBUTES,
**(contact_data.attributes or {}),
}
list_ids = (contact_data.list_ids or []) + settings.BREVO_API_CONTACT_LIST_IDS
contact = brevo_python.CreateContact(
email=contact_data.email,
attributes=attributes,
list_ids=list_ids,
update_enabled=contact_data.update_enabled,
)
api_configurations = {}
if timeout is not None:
api_configurations["_request_timeout"] = timeout
try:
response = contact_api.create_contact(contact, **api_configurations)
except brevo_python.rest.ApiException as err:
logger.exception("Failed to create contact in Brevo")
raise ContactCreationError("Failed to create contact in Brevo") from err
return response
@lru_cache(maxsize=1)
def get_marketing_service() -> MarketingServiceProtocol:
"""Return cached instance of configured marketing service."""
marketing_service_cls = import_string(settings.MARKETING_SERVICE_CLASS)
return marketing_service_cls()
@@ -1,15 +1,12 @@
"""Unit tests for the Authentication Backends."""
from unittest import mock
from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
from django.core.exceptions import SuspiciousOperation
import pytest
from core import models
from core.authentication.backends import OIDCAuthenticationBackend
from core.factories import UserFactory
from core.services import marketing_service
pytestmark = pytest.mark.django_db
@@ -415,139 +412,3 @@ def test_update_user_when_no_update_needed(django_assert_num_queries, claims):
user.refresh_from_db()
assert user.email == "john.doe@example.com"
@mock.patch.object(OIDCAuthenticationBackend, "signup_to_marketing_email")
def test_marketing_signup_new_user_enabled(mock_signup, monkeypatch, settings):
"""Test marketing signup for new user with settings enabled."""
settings.SIGNUP_NEW_USER_TO_MARKETING_EMAIL = True
klass = OIDCAuthenticationBackend()
email = "test@example.com"
def get_userinfo_mocked(*args):
return {"sub": "123", "email": email}
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
user = klass.get_or_create_user("test-token", None, None)
assert user.email == email
mock_signup.assert_called_once_with(email)
@mock.patch.object(OIDCAuthenticationBackend, "signup_to_marketing_email")
def test_marketing_signup_new_user_disabled(mock_signup, monkeypatch, settings):
"""Test no marketing signup for new user with settings disabled."""
settings.SIGNUP_NEW_USER_TO_MARKETING_EMAIL = False
klass = OIDCAuthenticationBackend()
email = "test@example.com"
def get_userinfo_mocked(*args):
return {"sub": "123", "email": email}
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
user = klass.get_or_create_user("test-token", None, None)
assert user.email == email
mock_signup.assert_not_called()
@mock.patch.object(OIDCAuthenticationBackend, "signup_to_marketing_email")
def test_marketing_signup_new_user_default_disabled(mock_signup, monkeypatch):
"""Test no marketing signup for new user with settings by default disabled."""
klass = OIDCAuthenticationBackend()
email = "test@example.com"
def get_userinfo_mocked(*args):
return {"sub": "123", "email": email}
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
user = klass.get_or_create_user("test-token", None, None)
assert user.email == email
mock_signup.assert_not_called()
@pytest.mark.parametrize(
"is_signup_enabled",
[True, False],
)
@mock.patch.object(OIDCAuthenticationBackend, "signup_to_marketing_email")
def test_marketing_signup_existing_user(
mock_signup, monkeypatch, settings, is_signup_enabled
):
"""Test no marketing signup for existing user regardless of settings."""
settings.SIGNUP_NEW_USER_TO_MARKETING_EMAIL = is_signup_enabled
klass = OIDCAuthenticationBackend()
db_user = UserFactory(email="test@example.com")
def get_userinfo_mocked(*args):
return {"sub": db_user.sub, "email": db_user.email}
monkeypatch.setattr(OIDCAuthenticationBackend, "get_userinfo", get_userinfo_mocked)
user = klass.get_or_create_user("test-token", None, None)
assert user == db_user
mock_signup.assert_not_called()
@mock.patch("core.authentication.backends.get_marketing_service")
def test_signup_to_marketing_email_success(mock_marketing):
"""Test successful marketing signup."""
email = "test@example.com"
# Call the method
OIDCAuthenticationBackend.signup_to_marketing_email(email)
# Verify service interaction
mock_service = mock_marketing.return_value
mock_service.create_contact.assert_called_once()
@pytest.mark.parametrize(
"error",
[
ImportError,
ImproperlyConfigured,
],
)
@mock.patch("core.authentication.backends.get_marketing_service")
def test_marketing_signup_handles_service_initialization_errors(
mock_marketing, error, settings
):
"""Tests errors that occur when trying to get/initialize the marketing service."""
settings.SIGNUP_NEW_USER_TO_MARKETING_EMAIL = True
mock_marketing.side_effect = error
# Should not raise any exception
OIDCAuthenticationBackend.signup_to_marketing_email("test@example.com")
@pytest.mark.parametrize(
"error",
[
marketing_service.ContactCreationError,
ImproperlyConfigured,
ImportError,
],
)
@mock.patch("core.authentication.backends.get_marketing_service")
def test_marketing_signup_handles_contact_creation_errors(
mock_marketing, error, settings
):
"""Tests errors that occur during the contact creation process."""
settings.SIGNUP_NEW_USER_TO_MARKETING_EMAIL = True
mock_marketing.return_value.create_contact.side_effect = error
# Should not raise any exception
OIDCAuthenticationBackend.signup_to_marketing_email("test@example.com")
@@ -1,187 +0,0 @@
"""
Test marketing services.
"""
# pylint: disable=W0621,W0613
from unittest import mock
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
import brevo_python
import pytest
from core.services.marketing_service import (
BrevoMarketingService,
ContactCreationError,
ContactData,
get_marketing_service,
)
def test_init_missing_api_key(settings):
"""Test initialization with missing API key."""
settings.BREVO_API_KEY = None
with pytest.raises(ImproperlyConfigured, match="Brevo API key is required"):
BrevoMarketingService()
def test_create_contact_missing_list_ids(settings):
"""Test contact creation with missing list IDs."""
settings.BREVO_API_KEY = "test-api-key"
settings.BREVO_API_CONTACT_LIST_IDS = None
settings.BREVO_API_CONTACT_ATTRIBUTES = {"source": "test"}
valid_contact_data = ContactData(
email="test@example.com",
attributes={"first_name": "Test"},
list_ids=[1, 2],
update_enabled=True,
)
brevo_service = BrevoMarketingService()
with pytest.raises(
ImproperlyConfigured, match="Default Brevo List IDs must be configured"
):
brevo_service.create_contact(valid_contact_data)
@mock.patch("brevo_python.ContactsApi")
def test_create_contact_success(mock_contact_api):
"""Test successful contact creation."""
mock_api = mock_contact_api.return_value
settings.BREVO_API_KEY = "test-api-key"
settings.BREVO_API_CONTACT_LIST_IDS = [1, 2, 3, 4]
settings.BREVO_API_CONTACT_ATTRIBUTES = {"source": "test"}
valid_contact_data = ContactData(
email="test@example.com",
attributes={"first_name": "Test"},
list_ids=[1, 2],
update_enabled=True,
)
brevo_service = BrevoMarketingService()
mock_api.create_contact.return_value = {"id": "test-id"}
response = brevo_service.create_contact(valid_contact_data)
assert response == {"id": "test-id"}
mock_api.create_contact.assert_called_once()
contact_arg = mock_api.create_contact.call_args[0][0]
assert contact_arg.email == "test@example.com"
assert contact_arg.attributes == {
**settings.BREVO_API_CONTACT_ATTRIBUTES,
**valid_contact_data.attributes,
}
assert set(contact_arg.list_ids) == {1, 2, 3, 4}
assert contact_arg.update_enabled is True
@mock.patch("brevo_python.ContactsApi")
def test_create_contact_with_timeout(mock_contact_api):
"""Test contact creation with timeout."""
mock_api = mock_contact_api.return_value
settings.BREVO_API_KEY = "test-api-key"
settings.BREVO_API_CONTACT_LIST_IDS = [1, 2, 3, 4]
settings.BREVO_API_CONTACT_ATTRIBUTES = {"source": "test"}
valid_contact_data = ContactData(
email="test@example.com",
attributes={"first_name": "Test"},
list_ids=[1, 2],
update_enabled=True,
)
brevo_service = BrevoMarketingService()
brevo_service.create_contact(valid_contact_data, timeout=30)
mock_api.create_contact.assert_called_once()
assert mock_api.create_contact.call_args[1]["_request_timeout"] == 30
@mock.patch("brevo_python.ContactsApi")
def test_create_contact_api_error(mock_contact_api):
"""Test contact creation API error handling."""
mock_api = mock_contact_api.return_value
settings.BREVO_API_KEY = "test-api-key"
settings.BREVO_API_CONTACT_LIST_IDS = [1, 2, 3, 4]
settings.BREVO_API_CONTACT_ATTRIBUTES = {"source": "test"}
valid_contact_data = ContactData(
email="test@example.com",
attributes={"first_name": "Test"},
list_ids=[1, 2],
update_enabled=True,
)
brevo_service = BrevoMarketingService()
mock_api.create_contact.side_effect = brevo_python.rest.ApiException()
with pytest.raises(ContactCreationError, match="Failed to create contact in Brevo"):
brevo_service.create_contact(valid_contact_data)
@pytest.fixture
def clear_marketing_cache():
"""Clear marketing service cache between tests."""
get_marketing_service.cache_clear()
yield
get_marketing_service.cache_clear()
def test_get_marketing_service_caching(clear_marketing_cache):
"""Test marketing service caching behavior."""
settings.BREVO_API_KEY = "test-api-key"
settings.MARKETING_SERVICE_CLASS = (
"core.services.marketing_service.BrevoMarketingService"
)
service1 = get_marketing_service()
service2 = get_marketing_service()
assert service1 is service2
assert isinstance(service1, BrevoMarketingService)
def test_get_marketing_service_invalid_class(clear_marketing_cache):
"""Test handling of invalid service class."""
settings.MARKETING_SERVICE_CLASS = "invalid.service.path"
with pytest.raises(ImportError):
get_marketing_service()
@mock.patch("core.services.marketing_service.import_string")
def test_service_instantiation_called_once(mock_import_string, clear_marketing_cache):
"""Test service class is instantiated only once."""
settings.BREVO_API_KEY = "test-api-key"
settings.MARKETING_SERVICE_CLASS = (
"core.services.marketing_service.BrevoMarketingService"
)
get_marketing_service.cache_clear()
mock_service_cls = mock.Mock()
mock_service_instance = mock.Mock()
mock_service_cls.return_value = mock_service_instance
mock_import_string.return_value = mock_service_cls
service1 = get_marketing_service()
service2 = get_marketing_service()
mock_import_string.assert_called_once_with(settings.MARKETING_SERVICE_CLASS)
mock_service_cls.assert_called_once()
assert service1 is service2
assert service1 is mock_service_instance
-41
View File
@@ -14,47 +14,6 @@ from django.conf import settings
from livekit.api import AccessToken, VideoGrants
import secrets
import string
from django.core.cache import cache
from cryptography.fernet import Fernet
import base64
def generate_random_passphrase(length=26):
"""Generate a random passphrase using letters and digits"""
alphabet = string.ascii_letters + string.digits
return ''.join(secrets.choice(alphabet) for _ in range(length))
def build_room_passphrase_key(room_id: str) -> str:
"""Build cache key for room passphrase."""
return f"room_passphrase:{room_id}"
def get_cached_passphrase(room_id: str) -> str:
"""Get or generate encrypted passphrase for a room.
Retrieves existing passphrase from cache or generates,
encrypts and caches a new one if not found.
"""
cypher = Fernet(settings.PASSPHRASE_ENCRYPTION_KEY.encode())
cache_key = build_room_passphrase_key(room_id)
encrypted_passphrase = cache.get(cache_key)
if encrypted_passphrase is None:
passphrase = generate_random_passphrase()
encrypted_passphrase = cypher.encrypt(passphrase.encode()).decode()
cache.set(cache_key, encrypted_passphrase, timeout=86400) # 24 hours
return passphrase
return cypher.decrypt(encrypted_passphrase.encode()).decode()
def clear_room_passphrase(room_id: str) -> None:
"""Remove room passphrase from cache."""
cache.delete(build_room_passphrase_key(room_id))
def generate_color(identity: str) -> str:
"""Generates a consistent HSL color based on a given identity string.
-28
View File
@@ -457,34 +457,6 @@ class Base(Configuration):
None, environ_name="SUMMARY_SERVICE_API_TOKEN", environ_prefix=None
)
# Marketing and communication settings
SIGNUP_NEW_USER_TO_MARKETING_EMAIL = values.BooleanValue(
False,
environ_name="SIGNUP_NEW_USERS_TO_NEWSLETTER",
environ_prefix=None,
help_text=(
"When enabled, new users are automatically added to mailing list "
"for product updates, marketing communications, and customized emails. "
),
)
MARKETING_SERVICE_CLASS = values.Value(
"core.services.marketing_service.BrevoMarketingService",
environ_name="MARKETING_SERVICE_CLASS",
environ_prefix=None,
)
BREVO_API_KEY = values.Value(
None, environ_name="BREVO_API_KEY", environ_prefix=None
)
BREVO_API_CONTACT_LIST_IDS = values.ListValue(
[],
environ_name="BREVO_API_CONTACT_LIST_IDS",
environ_prefix=None,
converter=lambda x: int(x), # pylint: disable=unnecessary-lambda
)
BREVO_API_CONTACT_ATTRIBUTES = values.DictValue({"VISIO_USER": True})
PASSPHRASE_ENCRYPTION_KEY = values.Value(environ_name="PASSPHRASE_ENCRYPTION_KEY", environ_prefix=None)
# pylint: disable=invalid-name
@property
def ENVIRONMENT(self):
-1
View File
@@ -27,7 +27,6 @@ requires-python = ">=3.10"
dependencies = [
"boto3==1.35.90",
"Brotli==1.1.0",
"brevo-python==1.1.2",
"celery[redis]==5.4.0",
"django-configurations==2.5.1",
"django-cors-headers==4.6.0",
@@ -8,7 +8,6 @@ export type ApiRoom = {
url: string
room: string
token: string
passphrase: string
}
configuration?: {
[key: string]: string | number | boolean
@@ -1,13 +1,8 @@
import { useEffect, useMemo, useRef, useState } from 'react'
import { useEffect, useMemo, useState } from 'react'
import { useQuery } from '@tanstack/react-query'
import { useTranslation } from 'react-i18next'
import { LiveKitRoom, type LocalUserChoices } from '@livekit/components-react'
import {
Room,
RoomOptions,
ExternalE2EEKeyProvider,
DeviceUnsupportedError,
} from 'livekit-client'
import { Room, RoomOptions } from 'livekit-client'
import { keys } from '@/api/queryKeys'
import { queryClient } from '@/api/queryClient'
import { Screen } from '@/layout/Screen'
@@ -22,9 +17,6 @@ import { VideoConference } from '../livekit/prefabs/VideoConference'
import posthog from 'posthog-js'
import { css } from '@/styled-system/css'
// todo - release worker when quitting the room, same for the key provider?
// todo - check, seems the demo app from livekit trigger the web worker twice because of re-rendering
export const Conference = ({
roomId,
userConfig,
@@ -71,49 +63,12 @@ export const Conference = ({
retry: false,
})
const e2eeEnabled = true
const workerRef = useRef<Worker | null>(null)
const keyProvider = useRef<any | null>(null)
const getKeyProvider = () => {
if (!keyProvider.current && typeof window !== 'undefined') {
keyProvider.current = new ExternalE2EEKeyProvider()
}
return keyProvider.current
}
const getWorker = () => {
if (!e2eeEnabled) {
return
}
if (!workerRef.current && typeof window !== 'undefined') {
workerRef.current = new Worker(
new URL('livekit-client/e2ee-worker', import.meta.url)
)
}
return workerRef.current
}
const e2eePassphrase = data?.livekit?.passphrase
const [e2eeSetupComplete, setE2eeSetupComplete] = useState(false)
const roomOptions = useMemo((): RoomOptions => {
const worker = getWorker()
const keyProvider = getKeyProvider()
// todo - explain why
const videoCodec = e2eeEnabled ? undefined : 'vp9'
const e2ee = e2eeEnabled ? { keyProvider, worker } : undefined
return {
adaptiveStream: true,
dynacast: true,
publishDefaults: {
// todo - explain why
red: !e2eeEnabled,
videoCodec,
videoCodec: 'vp9',
},
videoCaptureDefaults: {
deviceId: userConfig.videoDeviceId ?? undefined,
@@ -121,35 +76,12 @@ export const Conference = ({
audioCaptureDefaults: {
deviceId: userConfig.audioDeviceId ?? undefined,
},
e2ee,
}
// do not rely on the userConfig object directly as its reference may change on every render
}, [userConfig.videoDeviceId, userConfig.audioDeviceId])
const room = useMemo(() => new Room(roomOptions), [roomOptions])
useEffect(() => {
console.log('enter', e2eePassphrase)
if (e2eePassphrase) {
const keyProvider = getKeyProvider()
keyProvider
.setKey(e2eePassphrase)
.then(() => {
room.setE2EEEnabled(true).catch((e) => {
if (e instanceof DeviceUnsupportedError) {
alert(
`You're trying to join an encrypted meeting, but your browser does not support it. Please update it to the latest version and try again.`
)
console.error(e)
} else {
throw e
}
})
})
.then(() => setE2eeSetupComplete(true))
}
}, [room, e2eePassphrase])
const [showInviteDialog, setShowInviteDialog] = useState(mode === 'create')
const { t } = useTranslation('rooms')
@@ -170,10 +102,6 @@ export const Conference = ({
peerConnectionTimeout: 60000, // Default: 15s. Extended for slow TURN/TLS negotiation
}
const handleEncryptionError = () => {
console.log('error')
}
return (
<QueryAware status={isFetchError ? createStatus : fetchStatus}>
<Screen header={false} footer={false}>
@@ -181,14 +109,13 @@ export const Conference = ({
room={room}
serverUrl={data?.livekit?.url}
token={data?.livekit?.token}
connect={e2eeSetupComplete}
connect={true}
audio={userConfig.audioEnabled}
video={userConfig.videoEnabled}
connectOptions={connectOptions}
className={css({
backgroundColor: 'primaryDark.50 !important',
})}
onEncryptionError={handleEncryptionError}
>
<VideoConference />
{showInviteDialog && (
@@ -5,11 +5,9 @@ import { Text, P, ToggleButton, Div, H } from '@/primitives'
import { useTranslation } from 'react-i18next'
import { HStack, styled, VStack } from '@/styled-system/jsx'
import {
BackgroundBlur,
BackgroundOptions,
ProcessorWrapper,
BackgroundTransformer,
} from '@livekit/track-processors'
BackgroundBlurFactory,
BackgroundBlurProcessorInterface,
} from './blur/index'
const Information = styled('div', {
base: {
@@ -27,6 +25,8 @@ enum BlurRadius {
NORMAL = 10,
}
const isSupported = BackgroundBlurFactory.isSupported()
export const Effects = () => {
const { t } = useTranslation('rooms', { keyPrefix: 'effects' })
const { isCameraEnabled, cameraTrack, localParticipant } =
@@ -37,15 +37,12 @@ export const Effects = () => {
const localCameraTrack = cameraTrack?.track as LocalVideoTrack
const getProcessor = () => {
return localCameraTrack?.getProcessor() as ProcessorWrapper<BackgroundOptions>
return localCameraTrack?.getProcessor() as BackgroundBlurProcessorInterface
}
const getBlurRadius = (): BlurRadius => {
const processor = getProcessor()
return (
(processor?.transformer as BackgroundTransformer)?.options?.blurRadius ||
BlurRadius.NONE
)
return processor?.options.blurRadius || BlurRadius.NONE
}
const toggleBlur = async (blurRadius: number) => {
@@ -61,9 +58,11 @@ export const Effects = () => {
if (blurRadius == currentBlurRadius && processor) {
await localCameraTrack.stopProcessor()
} else if (!processor) {
await localCameraTrack.setProcessor(BackgroundBlur(blurRadius))
await localCameraTrack.setProcessor(
BackgroundBlurFactory.getProcessor({ blurRadius })!
)
} else {
await processor?.updateTransformerOptions({ blurRadius })
processor?.update({ blurRadius })
}
} catch (error) {
console.error('Error applying blur:', error)
@@ -147,7 +146,7 @@ export const Effects = () => {
>
{t('heading')}
</H>
{ProcessorWrapper.isSupported ? (
{isSupported ? (
<HStack>
<ToggleButton
size={'sm'}
@@ -15,9 +15,8 @@ import {
VideoTrack,
TrackRefContext,
ParticipantContextIfNeeded,
useIsSpeaking,
} from '@livekit/components-react'
import React, { useEffect } from 'react'
import React from 'react'
import {
isTrackReference,
isTrackReferencePinned,
@@ -0,0 +1,281 @@
import { ProcessorOptions, Track } from 'livekit-client'
import posthog from 'posthog-js'
import {
FilesetResolver,
ImageSegmenter,
ImageSegmenterResult,
} from '@mediapipe/tasks-vision'
import {
CLEAR_TIMEOUT,
SET_TIMEOUT,
TIMEOUT_TICK,
timerWorkerScript,
} from './TimerWorker'
import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'
const PROCESSING_WIDTH = 256
const PROCESSING_HEIGHT = 144
const SEGMENTATION_MASK_CANVAS_ID = 'background-blur-local-segmentation'
const BLUR_CANVAS_ID = 'background-blur-local'
const DEFAULT_BLUR = '10'
/**
* This implementation of video blurring is made to be run on CPU for browser that are
* not compatible with track-processor-js.
*
* It also make possible to run blurring on browser that does not implement MediaStreamTrackGenerator and
* MediaStreamTrackProcessor.
*/
export class BackgroundBlurCustomProcessor
implements BackgroundBlurProcessorInterface
{
options: BackgroundOptions
name: string
processedTrack?: MediaStreamTrack | undefined
source?: MediaStreamTrack
sourceSettings?: MediaTrackSettings
videoElement?: HTMLVideoElement
videoElementLoaded?: boolean
// Canvas containg the video processing result, of which we extract as stream.
outputCanvas?: HTMLCanvasElement
outputCanvasCtx?: CanvasRenderingContext2D
imageSegmenter?: ImageSegmenter
imageSegmenterResult?: ImageSegmenterResult
// Canvas used for resizing video source and projecting mask.
segmentationMaskCanvas?: HTMLCanvasElement
segmentationMaskCanvasCtx?: CanvasRenderingContext2D
// Mask containg the inference result.
segmentationMask?: ImageData
// The resized image of the video source.
sourceImageData?: ImageData
timerWorker?: Worker
constructor(opts: BackgroundOptions) {
this.name = 'blur'
this.options = opts
}
static get isSupported() {
return navigator.userAgent.toLowerCase().includes('firefox')
}
async init(opts: ProcessorOptions<Track.Kind>) {
if (!opts.element) {
throw new Error('Element is required for processing')
}
this.source = opts.track as MediaStreamTrack
this.sourceSettings = this.source!.getSettings()
this.videoElement = opts.element as HTMLVideoElement
this._createMainCanvas()
this._createMaskCanvas()
const stream = this.outputCanvas!.captureStream()
const tracks = stream.getVideoTracks()
if (tracks.length == 0) {
throw new Error('No tracks found for processing')
}
this.processedTrack = tracks[0]
this.segmentationMask = new ImageData(PROCESSING_WIDTH, PROCESSING_HEIGHT)
await this.initSegmenter()
this._initWorker()
posthog.capture('firefox-blurring-init')
}
update(opts: BackgroundOptions): void {
this.options = opts
}
_initWorker() {
this.timerWorker = new Worker(timerWorkerScript, {
name: 'Blurring',
})
this.timerWorker.onmessage = (data) => this.onTimerMessage(data)
// When hiding camera then showing it again, the onloadeddata callback is not fired again.
if (this.videoElementLoaded) {
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
})
} else {
this.videoElement!.onloadeddata = () => {
this.videoElementLoaded = true
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
})
}
}
}
onTimerMessage(response: { data: { id: number } }) {
if (response.data.id === TIMEOUT_TICK) {
this.process()
}
}
async initSegmenter() {
const vision = await FilesetResolver.forVisionTasks(
'https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision/wasm'
)
this.imageSegmenter = await ImageSegmenter.createFromOptions(vision, {
baseOptions: {
modelAssetPath:
'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter_landscape/float16/latest/selfie_segmenter_landscape.tflite',
delegate: 'CPU', // Use CPU for Firefox.
},
runningMode: 'VIDEO',
outputCategoryMask: true,
outputConfidenceMasks: false,
})
}
/**
* Resize the source video to the processing resolution.
*/
async sizeSource() {
this.segmentationMaskCanvasCtx?.drawImage(
this.videoElement!,
0,
0,
this.videoElement!.videoWidth,
this.videoElement!.videoHeight,
0,
0,
PROCESSING_WIDTH,
PROCESSING_HEIGHT
)
this.sourceImageData = this.segmentationMaskCanvasCtx?.getImageData(
0,
0,
PROCESSING_WIDTH,
PROCESSING_WIDTH
)
}
/**
* Run the segmentation.
*/
async segment() {
const startTimeMs = performance.now()
return new Promise<void>((resolve) => {
this.imageSegmenter!.segmentForVideo(
this.sourceImageData!,
startTimeMs,
(result: ImageSegmenterResult) => {
this.imageSegmenterResult = result
resolve()
}
)
})
}
/**
* TODO: future improvement with WebGL.
*/
async blur() {
const mask = this.imageSegmenterResult!.categoryMask!.getAsUint8Array()
for (let i = 0; i < mask.length; ++i) {
this.segmentationMask!.data[i * 4 + 3] = 255 - mask[i]
}
this.segmentationMaskCanvasCtx!.putImageData(this.segmentationMask!, 0, 0)
this.outputCanvasCtx!.globalCompositeOperation = 'copy'
this.outputCanvasCtx!.filter = 'blur(8px)'
this.outputCanvasCtx!.drawImage(
this.segmentationMaskCanvas!,
0,
0,
PROCESSING_WIDTH,
PROCESSING_HEIGHT,
0,
0,
this.videoElement!.videoWidth,
this.videoElement!.videoHeight
)
this.outputCanvasCtx!.globalCompositeOperation = 'source-in'
this.outputCanvasCtx!.filter = 'none'
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
this.outputCanvasCtx!.globalCompositeOperation = 'destination-over'
this.outputCanvasCtx!.filter = `blur(${this.options.blurRadius ?? DEFAULT_BLUR}px)`
this.outputCanvasCtx!.drawImage(this.videoElement!, 0, 0)
}
async process() {
await this.sizeSource()
await this.segment()
await this.blur()
this.timerWorker!.postMessage({
id: SET_TIMEOUT,
timeMs: 1000 / 30,
})
}
_createMainCanvas() {
this.outputCanvas = document.querySelector(
'canvas#background-blur-local'
) as HTMLCanvasElement
if (!this.outputCanvas) {
this.outputCanvas = this._createCanvas(
BLUR_CANVAS_ID,
this.sourceSettings!.width!,
this.sourceSettings!.height!
)
}
this.outputCanvasCtx = this.outputCanvas.getContext('2d')!
}
_createMaskCanvas() {
this.segmentationMaskCanvas = document.querySelector(
`#${SEGMENTATION_MASK_CANVAS_ID}`
) as HTMLCanvasElement
if (!this.segmentationMaskCanvas) {
this.segmentationMaskCanvas = this._createCanvas(
SEGMENTATION_MASK_CANVAS_ID,
PROCESSING_WIDTH,
PROCESSING_HEIGHT
)
}
this.segmentationMaskCanvasCtx =
this.segmentationMaskCanvas.getContext('2d')!
}
_createCanvas(id: string, width: number, height: number) {
const element = document.createElement('canvas')
element.setAttribute('id', id)
element.setAttribute('width', '' + width)
element.setAttribute('height', '' + height)
return element
}
async restart(opts: ProcessorOptions<Track.Kind>) {
await this.destroy()
return this.init(opts)
}
async destroy() {
this.timerWorker?.postMessage({
id: CLEAR_TIMEOUT,
})
this.timerWorker?.terminate()
this.imageSegmenter?.close()
}
}
@@ -0,0 +1,48 @@
import {
BackgroundBlur,
BackgroundTransformer,
ProcessorWrapper,
} from '@livekit/track-processors'
import { ProcessorOptions, Track } from 'livekit-client'
import { BackgroundBlurProcessorInterface, BackgroundOptions } from '.'
/**
* This is simply a wrapper around track-processor-js Processor
* in order to be compatible with a common interface BackgroundBlurProcessorInterface
* used accross the project.
*/
export class BackgroundBlurTrackProcessorJsWrapper
implements BackgroundBlurProcessorInterface
{
name: string = 'Blur'
processor: ProcessorWrapper<BackgroundOptions>
constructor(opts: BackgroundOptions) {
this.processor = BackgroundBlur(opts.blurRadius)
}
async init(opts: ProcessorOptions<Track.Kind>) {
return this.processor.init(opts)
}
async restart(opts: ProcessorOptions<Track.Kind>) {
return this.processor.restart(opts)
}
async destroy() {
return this.processor.destroy()
}
update(opts: BackgroundOptions): void {
this.processor.updateTransformerOptions(opts)
}
get processedTrack() {
return this.processor.processedTrack
}
get options() {
return (this.processor.transformer as BackgroundTransformer).options
}
}
@@ -0,0 +1,71 @@
/**
* From https://github.com/jitsi/jitsi-meet
*/
/**
* SET_TIMEOUT constant is used to set interval and it is set in
* the id property of the request.data property. TimeMs property must
* also be set.
*
* ```
* //Request.data example:
* {
* id: SET_TIMEOUT,
* timeMs: 33
* }
* ```
*/
export const SET_TIMEOUT = 1
/**
* CLEAR_TIMEOUT constant is used to clear the interval and it is set in
* the id property of the request.data property.
*
* ```
* {
* id: CLEAR_TIMEOUT
* }
* ```
*/
export const CLEAR_TIMEOUT = 2
/**
* TIMEOUT_TICK constant is used as response and it is set in the id property.
*
* ```
* {
* id: TIMEOUT_TICK
* }
* ```
*/
export const TIMEOUT_TICK = 3
/**
* The following code is needed as string to create a URL from a Blob.
* The URL is then passed to a WebWorker. Reason for this is to enable
* use of setInterval that is not throttled when tab is inactive.
*/
const code = `
var timer;
onmessage = function(request) {
switch (request.data.id) {
case ${SET_TIMEOUT}: {
timer = setTimeout(() => {
postMessage({ id: ${TIMEOUT_TICK} });
}, request.data.timeMs);
break;
}
case ${CLEAR_TIMEOUT}: {
if (timer) {
clearTimeout(timer);
}
break;
}
}
};
`
export const timerWorkerScript = URL.createObjectURL(
new Blob([code], { type: 'application/javascript' })
)
@@ -0,0 +1,32 @@
import { ProcessorWrapper } from '@livekit/track-processors'
import { Track, TrackProcessor } from 'livekit-client'
import { BackgroundBlurTrackProcessorJsWrapper } from './BackgroundBlurTrackProcessorJsWrapper'
import { BackgroundBlurCustomProcessor } from './BackgroundBlurCustomProcessor'
export type BackgroundOptions = {
blurRadius?: number
}
export interface BackgroundBlurProcessorInterface
extends TrackProcessor<Track.Kind> {
update(opts: BackgroundOptions): void
options: BackgroundOptions
}
export class BackgroundBlurFactory {
static isSupported() {
return (
ProcessorWrapper.isSupported || BackgroundBlurCustomProcessor.isSupported
)
}
static getProcessor(opts: BackgroundOptions) {
if (ProcessorWrapper.isSupported) {
return new BackgroundBlurTrackProcessorJsWrapper(opts)
}
if (BackgroundBlurCustomProcessor.isSupported) {
return new BackgroundBlurCustomProcessor(opts)
}
return null
}
}
+1 -1
View File
@@ -87,7 +87,7 @@
},
"effects": {
"activateCamera": "Your camera is disabled. Choose an option to enable it.",
"notAvailable": "The blur effect will be available soon on your browser. We're working on it! In the meantime, you can use Google Chrome :(",
"notAvailable": "The blur effect will be available soon on your browser. We're working on it! In the meantime, you can use Google Chrome for best performance or Firefox :(",
"heading": "Blur",
"blur": {
"light": "Light blur",
+1 -1
View File
@@ -87,7 +87,7 @@
},
"effects": {
"activateCamera": "Votre camera est désactivée. Choisissez une option pour l'activer.",
"notAvailable": "L'effet de flou sera bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome :(",
"notAvailable": "L'effet de flou sera bientôt disponible sur votre navigateur. Nous y travaillons ! En attendant, vous pouvez utiliser Google Chrome pour une meilleure performance ou Firefox :(",
"heading": "Flou",
"blur": {
"light": "Léger flou",
@@ -17,10 +17,6 @@ livekit:
udp_port: 443
domain: livekit.127.0.0.1.nip.io
loadBalancerAnnotations: {}
webhook:
api_key: devkey
urls:
- https://meet.127.0.0.1.nip.io/api/v1.0/rooms/livekit-webhook/
loadBalancer:
@@ -61,7 +61,6 @@ backend:
RECORDING_STORAGE_EVENT_TOKEN: password
SUMMARY_SERVICE_ENDPOINT: http://meet-summary:80/api/v1/tasks/
SUMMARY_SERVICE_API_TOKEN: password
PASSPHRASE_ENCRYPTION_KEY: lT3cX5dzFhCe-9xNjXUiTCX00r2ZgHgGUJKO66x-QIo=
migrate: