wip align env variable in metadata agent

src/agents/Dockerfile

@@ -21,4 +21,4 @@ COPY --from=builder /install /usr/local
 
 COPY . .
 
-CMD ["python", "multi-user-transcriber.py", "start"]
+CMD ["python", "multi_user_transcriber.py", "start"]

src/agents/metadata_extractor.py

@@ -0,0 +1,391 @@
+"""Metadata agent that extracts metadata from active room."""
+
+import asyncio
+import json
+import logging
+import os
+from dataclasses import asdict, dataclass
+from datetime import datetime, timezone
+from io import BytesIO
+from typing import List, Optional
+
+from dotenv import load_dotenv
+from livekit import api, rtc
+from livekit.agents import (
+    Agent,
+    AgentSession,
+    AutoSubscribe,
+    JobContext,
+    JobProcess,
+    JobRequest,
+    RoomInputOptions,
+    RoomIO,
+    RoomOutputOptions,
+    WorkerOptions,
+    WorkerPermissions,
+    cli,
+    utils,
+)
+from livekit.plugins import silero
+from minio import Minio
+from minio.error import S3Error
+
+load_dotenv()
+
+logger = logging.getLogger("metadata-extractor")
+
+AGENT_NAME = os.getenv("ROOM_METADATA_EXTRACTOR_AGENT_NAME", "metadata-extractor")
+
+
+@dataclass
+class MetadataEvent:
+    """Wip."""
+
+    participant_id: str
+    type: str
+    timestamp: datetime
+    data: Optional[str] = None
+
+    def serialize(self) -> dict:
+        """Return a JSON-serializable dictionary representation of the event."""
+        data = asdict(self)
+        data["timestamp"] = self.timestamp.isoformat()
+        return data
+
+
+class VADAgent(Agent):
+    """Agent that monitors voice activity for a specific participant."""
+
+    def __init__(self, participant_identity: str, events: List):
+        """Wip."""
+        super().__init__(
+            instructions="not-needed",
+        )
+        self.participant_identity = participant_identity
+        self.events = events
+
+    async def on_enter(self) -> None:
+        """Initialize VAD monitoring for this participant."""
+
+        @self.session.on("user_state_changed")
+        def on_user_state(event):
+            timestamp = datetime.now(timezone.utc)
+
+            if event.new_state == "speaking":
+                event = MetadataEvent(
+                    participant_id=self.participant_identity,
+                    type="speech_start",
+                    timestamp=timestamp,
+                )
+                self.events.append(event)
+
+            elif event.old_state == "speaking":
+                event = MetadataEvent(
+                    participant_id=self.participant_identity,
+                    type="speech_end",
+                    timestamp=timestamp,
+                )
+                self.events.append(event)
+
+
+class MetadataAgent:
+    """Monitor and manage real-time metadata extraction from meeting rooms.
+
+    Oversees VAD (Voice Activity Detection) and participant metadata streams
+    to track and analyze real-time events, coordinating data collection across
+    participants for insights like speaking activity and engagement.
+    """
+
+    def __init__(self, ctx: JobContext, recording_id: str):
+        """Initialize metadata agent."""
+        self.minio_client = Minio(
+            endpoint=os.getenv("AWS_S3_ENDPOINT_URL"),
+            access_key=os.getenv("AWS_S3_ACCESS_KEY_ID"),
+            secret_key=os.getenv("AWS_S3_SECRET_ACCESS_KEY"),
+            secure=os.getenv("AWS_S3_SECURE_ACCESS", "False").lower() == "true",
+        )
+
+        # todo - raise error if none
+        self.bucket_name = os.getenv("AWS_STORAGE_BUCKET_NAME")
+
+        self.ctx = ctx
+        self._sessions: dict[str, AgentSession] = {}
+        self._tasks: set[asyncio.Task] = set()
+
+        self.output_filename = (
+            f"{os.getenv('AWS_S3_OUTPUT_FOLDER', 'metadata')}/{recording_id}-metadata.json"
+        )
+
+        # Storage for events
+        self.events = []
+        self.participants = {}
+
+        logger.info("MetadataAgent initialized")
+
+    def start(self):
+        """Start listening for participant connection events."""
+        self.ctx.room.on("participant_connected", self.on_participant_connected)
+        self.ctx.room.on("participant_disconnected", self.on_participant_disconnected)
+        self.ctx.room.on("participant_name_changed", self.on_participant_name_changed)
+
+        self.ctx.room.register_text_stream_handler("lk.chat", self.handle_chat_stream)
+
+        logger.info("Started listening for participant events")
+
+    async def on_chat_message_received(
+        self, reader: rtc.TextStreamReader, participant_identity: str
+    ):
+        """Wip."""
+        full_text = await reader.read_all()
+        logger.info(
+            "Received chat message from %s: '%s'", participant_identity, full_text
+        )
+
+        self.events.append(
+            MetadataEvent(
+                participant_id=participant_identity,
+                type="chat_received",
+                timestamp=datetime.now(timezone.utc),
+                data=full_text,
+            )
+        )
+
+    def handle_chat_stream(self, reader, participant_identity):
+        """Wip."""
+        task = asyncio.create_task(
+            self.on_chat_message_received(reader, participant_identity)
+        )
+        self._tasks.add(task)
+        task.add_done_callback(lambda _: self._tasks.remove(task))
+
+    def save(self):
+        """Wip."""
+        logger.info("Persisting processed metadata output to disk…")
+
+        participants = []
+        for k, v in self.participants.items():
+            participants.append({"participantId": k, "name": v})
+
+        sorted_event = sorted(self.events, key=lambda e: e.timestamp)
+
+        payload = {
+            "events": [event.serialize() for event in sorted_event],
+            "participants": participants,
+        }
+
+        data = json.dumps(payload, indent=2).encode("utf-8")
+        stream = BytesIO(data)
+
+        try:
+            self.minio_client.put_object(
+                self.bucket_name,
+                self.output_filename,
+                stream,
+                length=len(data),
+                content_type="application/json",
+            )
+            logger.info(
+                "Uploaded speaker meeting metadata",
+            )
+        except S3Error:
+            logger.exception(
+                "Failed to upload meeting metadata",
+            )
+
+    async def aclose(self):
+        """Close all sessions and cleanup resources."""
+        logger.info("Closing all VAD monitoring sessions…")
+
+        await utils.aio.cancel_and_wait(*self._tasks)
+
+        await asyncio.gather(
+            *[self._close_session(session) for session in self._sessions.values()],
+            return_exceptions=True,
+        )
+
+        self.ctx.room.off("participant_connected", self.on_participant_connected)
+        self.ctx.room.off("participant_disconnected", self.on_participant_disconnected)
+        self.ctx.room.off("participant_name_changed", self.on_participant_name_changed)
+
+        logger.info("All VAD sessions closed")
+        self.save()
+
+    def on_participant_connected(self, participant: rtc.RemoteParticipant):
+        """Handle new participant connection by starting VAD monitoring."""
+        if participant.identity in self._sessions:
+            logger.debug("Session already exists for %s", participant.identity)
+            return
+
+        self.events.append(
+            MetadataEvent(
+                participant_id=participant.identity,
+                type="participant_connected",
+                timestamp=datetime.now(timezone.utc),
+            )
+        )
+
+        self.participants[participant.identity] = participant.name
+
+        logger.info("New participant connected: %s", participant.identity)
+        task = asyncio.create_task(self._start_session(participant))
+        self._tasks.add(task)
+
+        def on_task_done(task: asyncio.Task):
+            try:
+                self._sessions[participant.identity] = task.result()
+            except Exception:
+                logger.exception("Failed to start session for %s", participant.identity)
+            finally:
+                self._tasks.discard(task)
+
+        task.add_done_callback(on_task_done)
+
+    def on_participant_disconnected(self, participant: rtc.RemoteParticipant):
+        """Handle participant disconnection by closing VAD monitoring."""
+        self.events.append(
+            MetadataEvent(
+                participant_id=participant.identity,
+                type="participant_disconnected",
+                timestamp=datetime.now(timezone.utc),
+            )
+        )
+
+        session = self._sessions.pop(participant.identity, None)
+        if session is None:
+            logger.debug("No session found for %s", participant.identity)
+            return
+
+        logger.info("Participant disconnected: %s", participant.identity)
+        task = asyncio.create_task(self._close_session(session))
+        self._tasks.add(task)
+
+        def on_close_done(_):
+            self._tasks.discard(task)
+            logger.info(
+                "VAD session closed for %s (remaining sessions: %d)",
+                participant.identity,
+                len(self._sessions),
+            )
+
+        task.add_done_callback(on_close_done)
+
+    def on_participant_name_changed(self, participant: rtc.RemoteParticipant):
+        """Wip."""
+        logger.info("Participant's name changed: %s", participant.identity)
+        self.participants[participant.identity] = participant.name
+
+    async def _start_session(self, participant: rtc.RemoteParticipant) -> AgentSession:
+        """Create and start VAD monitoring session for participant."""
+        if participant.identity in self._sessions:
+            return self._sessions[participant.identity]
+
+        # Create session with VAD only - no STT, LLM, or TTS
+        session = AgentSession(
+            vad=self.ctx.proc.userdata["vad"],
+            turn_detection="vad",
+            user_away_timeout=30.0,
+        )
+
+        # Set up room IO to receive audio from this specific participant
+        room_io = RoomIO(
+            agent_session=session,
+            room=self.ctx.room,
+            participant=participant,
+            input_options=RoomInputOptions(
+                audio_enabled=True,
+                text_enabled=False,
+            ),
+            output_options=RoomOutputOptions(
+                audio_enabled=False,
+                transcription_enabled=False,
+            ),
+        )
+
+        await room_io.start()
+        await session.start(
+            agent=VADAgent(
+                participant_identity=participant.identity, events=self.events
+            )
+        )
+
+        return session
+
+    async def _close_session(self, session: AgentSession) -> None:
+        """Close and cleanup VAD monitoring session."""
+        try:
+            await session.drain()
+            await session.aclose()
+        except Exception:
+            logger.exception("Error closing session")
+
+
+async def entrypoint(ctx: JobContext):
+    """Initialize and run the multi-user VAD monitor."""
+    logger.info("Starting metadata agent in room: %s", ctx.room.name)
+    recording_id = ctx.job.metadata
+    vad_monitor = MetadataAgent(ctx, recording_id)
+    vad_monitor.start()
+
+    # Connect to room and subscribe to audio only
+    await ctx.connect(auto_subscribe=AutoSubscribe.AUDIO_ONLY)
+
+    existing_participants = list(ctx.room.remote_participants.values())
+    for participant in existing_participants:
+        vad_monitor.on_participant_connected(participant)
+
+    async def cleanup():
+        logger.info("Shutting down VAD monitor...")
+        await vad_monitor.aclose()
+
+    ctx.add_shutdown_callback(cleanup)
+
+
+async def handle_job_request(job_req: JobRequest) -> None:
+    """Accept or reject the job request based on agent presence in the room."""
+    room_name = job_req.room.name
+    recording_id = job_req.job.metadata
+    agent_identity = f"{AGENT_NAME}-{room_name}"
+
+    async with api.LiveKitAPI() as lk:
+        try:
+            resp = await lk.room.list_participants(
+                list=api.ListParticipantsRequest(room=room_name)
+            )
+            already_present = any(
+                p.kind == rtc.ParticipantKind.PARTICIPANT_KIND_AGENT
+                and p.identity == agent_identity
+                for p in resp.participants
+            )
+            if already_present:
+                logger.info("Agent already in the room '%s' — reject", room_name)
+                await job_req.reject()
+            else:
+                logger.info(
+                    "Accept job for '%s' — identity=%s", room_name, agent_identity
+                )
+                await job_req.accept(identity=agent_identity, metadata=recording_id)
+        except Exception:
+            logger.exception("Error treating the job for '%s'", room_name)
+            await job_req.reject()
+
+
+def prewarm(proc: JobProcess):
+    """Preload voice activity detection model."""
+    proc.userdata["vad"] = silero.VAD.load()
+
+
+if __name__ == "__main__":
+    cli.run_app(
+        WorkerOptions(
+            entrypoint_fnc=entrypoint,
+            prewarm_fnc=prewarm,
+            request_fnc=handle_job_request,
+            agent_name=AGENT_NAME,
+            permissions=WorkerPermissions(
+                can_publish=False,
+                can_publish_data=False,
+                can_subscribe=True,
+                hidden=True,
+            ),
+        )
+    )

src/agents/pyproject.toml

@@ -7,7 +7,8 @@ dependencies = [
     "livekit-agents==1.2.6",
     "livekit-plugins-deepgram==1.2.6",
     "livekit-plugins-silero==1.2.6",
-    "python-dotenv==1.1.1"
+    "python-dotenv==1.1.1",
+    "minio==7.2.15"
 ]
 
 [project.optional-dependencies]

src/backend/core/api/viewsets.py

@@ -31,6 +31,10 @@ from core.recording.event.exceptions import (
 )
 from core.recording.event.notification import notification_service
 from core.recording.event.parsers import get_parser
+from core.recording.services.metadata_extractor import (
+    MetadataExtractorException,
+    MetadataExtractorService,
+)
 from core.recording.worker.exceptions import (
     RecordingStartError,
     RecordingStopError,
@@ -328,6 +332,15 @@ class RoomViewSet(
                 status=drf_status.HTTP_500_INTERNAL_SERVER_ERROR,
             )
 
+        if (
+            settings.ROOM_METADATA_EXTRACTOR_ENABLED
+            and recording.mode == models.RecordingModeChoices.TRANSCRIPT
+        ):
+            try:
+                MetadataExtractorService().start(recording)
+            except MetadataExtractorException:
+                pass
+
         return drf_response.Response(
             {"message": f"Recording successfully started for room {room.slug}"},
             status=drf_status.HTTP_201_CREATED,

src/backend/core/recording/services/metadata_extractor.py

@@ -0,0 +1,97 @@
+"""Wip."""
+
+from logging import getLogger
+
+from django.conf import settings
+
+from asgiref.sync import async_to_sync
+from livekit.protocol.agent_dispatch import (
+    CreateAgentDispatchRequest,
+)
+
+from core import utils
+
+logger = getLogger(__name__)
+
+
+class MetadataExtractorException(Exception):
+    """Wip."""
+
+
+class MetadataExtractorService:
+    """Wip."""
+
+    @async_to_sync
+    async def start(self, recording):
+        """Wip."""
+
+        lkapi = utils.create_livekit_client()
+        room_id = str(recording.room.id)
+
+        try:
+            response = await lkapi.agent_dispatch.create_dispatch(
+                CreateAgentDispatchRequest(
+                    agent_name=settings.ROOM_METADATA_EXTRACTOR_AGENT_NAME,
+                    room=room_id,
+                    metadata=str(recording.id),
+                )
+            )
+        except Exception as e:
+            logger.exception(
+                "Failed to create metadata extractor agent for room %s", room_id
+            )
+            raise MetadataExtractorException(
+                "Failed to create metadata extractor agent"
+            ) from e
+        finally:
+            await lkapi.aclose()
+
+        dispatch_id = getattr(response, "id", None)
+
+        if not dispatch_id:
+            logger.error("LiveKit response missing dispatch ID for room %s", room_id)
+            raise MetadataExtractorException(
+                f"LiveKit did not return a dispatch_id for room {room_id}"
+            )
+
+        return dispatch_id
+
+    @async_to_sync
+    async def stop(self, recording):
+        """Wip."""
+
+        room_name = str(recording.room.id)
+        lkapi = utils.create_livekit_client()
+
+        try:
+            dispatches = await lkapi.agent_dispatch.list_dispatch(room_name=room_name)
+
+            dispatch_id = next(
+                (
+                    d.id
+                    for d in dispatches
+                    if d.agent_name == settings.ROOM_METADATA_EXTRACTOR_AGENT_NAME
+                ),
+                None,
+            )
+
+            if not dispatch_id:
+                logger.warning(
+                    "No metadata extractor agent found for room %s", room_name
+                )
+                return None
+
+            await lkapi.agent_dispatch.delete_dispatch(
+                dispatch_id=str(dispatch_id), room_name=room_name
+            )
+
+        except Exception as e:
+            logger.exception(
+                "Failed to stop metadata extractor agent dispatch for room %s",
+                room_name,
+            )
+            raise MetadataExtractorException(
+                f"Failed to stop metadata metadata extractor agent for room {room_name}"
+            ) from e
+        finally:
+            await lkapi.aclose()

src/backend/core/services/livekit_events.py

@@ -11,6 +11,10 @@ from django.conf import settings
 from livekit import api
 
 from core import models
+from core.recording.services.metadata_extractor import (
+    MetadataExtractorException,
+    MetadataExtractorService,
+)
 from core.recording.services.recording_events import (
     RecordingEventsError,
     RecordingEventsService,
@@ -126,7 +130,7 @@ class LiveKitEventsService:
         """Handle 'egress_ended' event."""
 
         try:
-            recording = models.Recording.objects.get(
+            recording = models.Recording.objects.select_related("room").get(
                 worker_id=data.egress_info.egress_id
             )
         except models.Recording.DoesNotExist as err:
@@ -134,6 +138,15 @@ class LiveKitEventsService:
                 f"Recording with worker ID {data.egress_info.egress_id} does not exist"
             ) from err
 
+        if (
+            settings.ROOM_METADATA_EXTRACTOR_ENABLED
+            and recording.mode == models.RecordingModeChoices.TRANSCRIPT
+        ):
+            try:
+                MetadataExtractorService().stop(recording)
+            except MetadataExtractorException:
+                pass
+
         if (
             data.egress_info.status == api.EgressStatus.EGRESS_LIMIT_REACHED
             and recording.status == models.RecordingStatusChoices.ACTIVE

src/backend/meet/settings.py

@@ -668,6 +668,16 @@ class Base(Configuration):
         environ_prefix=None,
     )
 
+    # Metadata Extractor settings
+    ROOM_METADATA_EXTRACTOR_ENABLED = values.BooleanValue(
+        False, environ_name="ROOM_METADATA_EXTRACTOR_ENABLED", environ_prefix=None
+    )
+    ROOM_METADATA_EXTRACTOR_AGENT_NAME = values.Value(
+        "metadata-extractor",
+        environ_name="ROOM_METADATA_EXTRACTOR_AGENT_NAME",
+        environ_prefix=None,
+    )
+
     # External Applications
     APPLICATION_CLIENT_ID_LENGTH = values.PositiveIntegerValue(
         40,

src/frontend/src/features/rooms/livekit/hooks/useDeviceShortcut.ts

@@ -6,12 +6,12 @@ export const useDeviceShortcut = (kind: MediaDeviceKind) => {
     switch (kind) {
       case 'audioinput':
         return {
-          key: 'd',
+          key: 'e',
           ctrlKey: true,
         }
       case 'videoinput':
         return {
-          key: 'e',
+          key: 'd',
           ctrlKey: true,
         }
       default:

src/helm/env.d/dev-keycloak/values.meet.yaml.gotmpl

@@ -73,6 +73,7 @@ backend:
     ROOM_TELEPHONY_PHONE_NUMBER: '+33901020304'
     SSL_CERT_FILE: /usr/local/lib/python3.13/site-packages/certifi/cacert.pem
     ROOM_SUBTITLE_ENABLED: True
+    ROOM_METADATA_EXTRACTOR_ENABLED: True
 
 
   migrate:
@@ -262,12 +263,23 @@ agents:
     LIVEKIT_API_KEY: {{ $key }}
     {{- end }}
     {{- end }}
+    AWS_S3_ENDPOINT_URL: minio.meet.svc.cluster.local:9000
+    AWS_S3_ACCESS_KEY_ID: meet
+    AWS_S3_SECRET_ACCESS_KEY: password
+    AWS_S3_SECURE_ACCESS: False
+    AWS_STORAGE_BUCKET_NAME: meet-media-storage
+    AWS_S3_OUTPUT_FOLDER: recordings
 
   image:
     repository: localhost:5001/meet-agents
     pullPolicy: Always
     tag: "latest"
 
+  command:
+    - "python"
+    - "metadata_extractor.py"
+    - "start"
+
   # Extra volume mounts to manage our local custom CA and avoid to disable ssl
   extraVolumeMounts:
     - name: certs