Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| df90ea96e6 |
@@ -11,7 +11,6 @@ and this project adheres to
|
||||
### Added
|
||||
|
||||
- 👷(docker) add arm64 platform support for image builds
|
||||
- ✨(summary) add localization support for transcription context text
|
||||
|
||||
### Changed
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
Gitlint extra rule to validate that the message title is of the form
|
||||
"<gitmoji>(<scope>) <subject>"
|
||||
"""
|
||||
|
||||
from __future__ import unicode_literals
|
||||
|
||||
import re
|
||||
|
||||
@@ -167,7 +167,6 @@ class NotificationService:
|
||||
owner_access.user.timezone
|
||||
).strftime("%H:%M"),
|
||||
"download_link": f"{get_recording_download_base_url()}/{recording.id}",
|
||||
"context_language": owner_access.user.language,
|
||||
}
|
||||
|
||||
headers = {
|
||||
|
||||
@@ -15,8 +15,8 @@ from summary.core.config import get_settings
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
class TranscribeSummarizeTaskCreation(BaseModel):
|
||||
"""Transcription and summarization parameters."""
|
||||
class TaskCreation(BaseModel):
|
||||
"""Task data."""
|
||||
|
||||
owner_id: str
|
||||
filename: str
|
||||
@@ -28,7 +28,6 @@ class TranscribeSummarizeTaskCreation(BaseModel):
|
||||
recording_time: Optional[str]
|
||||
language: Optional[str]
|
||||
download_link: Optional[str]
|
||||
context_language: Optional[str] = None
|
||||
|
||||
@field_validator("language")
|
||||
@classmethod
|
||||
@@ -46,8 +45,8 @@ router = APIRouter(prefix="/tasks")
|
||||
|
||||
|
||||
@router.post("/")
|
||||
async def create_transcribe_summarize_task(request: TranscribeSummarizeTaskCreation):
|
||||
"""Create a transcription and summarization task."""
|
||||
async def create_task(request: TaskCreation):
|
||||
"""Create a task."""
|
||||
task = process_audio_transcribe_summarize_v2.apply_async(
|
||||
args=[
|
||||
request.owner_id,
|
||||
@@ -60,7 +59,6 @@ async def create_transcribe_summarize_task(request: TranscribeSummarizeTaskCreat
|
||||
request.recording_time,
|
||||
request.language,
|
||||
request.download_link,
|
||||
request.context_language,
|
||||
],
|
||||
queue=settings.transcribe_queue,
|
||||
)
|
||||
|
||||
@@ -112,16 +112,19 @@ class MetadataManager:
|
||||
if self._is_disabled or self.has_task_id(task_id):
|
||||
return
|
||||
|
||||
_, filename, email, _, received_at, *_ = task_args
|
||||
|
||||
start_time = time.time()
|
||||
initial_metadata = {
|
||||
"start_time": start_time,
|
||||
"start_time": time.time(),
|
||||
"asr_model": settings.whisperx_asr_model,
|
||||
"retries": 0,
|
||||
}
|
||||
|
||||
_, filename, email, _, received_at, *_ = task_args
|
||||
|
||||
initial_metadata = {
|
||||
**initial_metadata,
|
||||
"filename": filename,
|
||||
"email": email,
|
||||
"queuing_time": round(start_time - received_at, 2),
|
||||
"queuing_time": round(initial_metadata["start_time"] - received_at, 2),
|
||||
}
|
||||
|
||||
self._save_metadata(task_id, initial_metadata)
|
||||
|
||||
@@ -10,13 +10,14 @@ import openai
|
||||
import sentry_sdk
|
||||
from celery import Celery, signals
|
||||
from celery.utils.log import get_task_logger
|
||||
from requests import exceptions
|
||||
from requests import Session, exceptions
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util import Retry
|
||||
|
||||
from summary.core.analytics import MetadataManager, get_analytics
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.file_service import FileService, FileServiceException
|
||||
from summary.core.llm_service import LLMException, LLMObservability, LLMService
|
||||
from summary.core.locales import get_locale
|
||||
from summary.core.prompt import (
|
||||
FORMAT_NEXT_STEPS,
|
||||
FORMAT_PLAN,
|
||||
@@ -28,7 +29,6 @@ from summary.core.prompt import (
|
||||
PROMPT_USER_PART,
|
||||
)
|
||||
from summary.core.transcript_formatter import TranscriptFormatter
|
||||
from summary.core.webhook_service import submit_content
|
||||
|
||||
settings = get_settings()
|
||||
analytics = get_analytics()
|
||||
@@ -55,17 +55,89 @@ if settings.sentry_dsn and settings.sentry_is_enabled:
|
||||
sentry_sdk.init(dsn=settings.sentry_dsn, enable_tracing=True)
|
||||
|
||||
|
||||
file_service = FileService()
|
||||
file_service = FileService(logger=logger)
|
||||
|
||||
|
||||
def transcribe_audio(task_id, filename, language):
|
||||
"""Transcribe an audio file using WhisperX.
|
||||
def create_retry_session():
|
||||
"""Create an HTTP session configured with retry logic."""
|
||||
session = Session()
|
||||
retries = Retry(
|
||||
total=settings.webhook_max_retries,
|
||||
backoff_factor=settings.webhook_backoff_factor,
|
||||
status_forcelist=settings.webhook_status_forcelist,
|
||||
allowed_methods={"POST"},
|
||||
)
|
||||
session.mount("https://", HTTPAdapter(max_retries=retries))
|
||||
return session
|
||||
|
||||
Downloads the audio from MinIO, sends it to WhisperX for transcription,
|
||||
and tracks metadata throughout the process.
|
||||
|
||||
Returns the transcription object, or None if the file could not be retrieved.
|
||||
def format_actions(llm_output: dict) -> str:
|
||||
"""Format the actions from the LLM output into a markdown list.
|
||||
|
||||
fomat:
|
||||
- [ ] Action title Assignée à : assignee1, assignee2, Échéance : due_date
|
||||
"""
|
||||
lines = []
|
||||
for action in llm_output.get("actions", []):
|
||||
title = action.get("title", "").strip()
|
||||
assignees = ", ".join(action.get("assignees", [])) or "-"
|
||||
due_date = action.get("due_date") or "-"
|
||||
line = f"- [ ] {title} Assignée à : {assignees}, Échéance : {due_date}"
|
||||
lines.append(line)
|
||||
if lines:
|
||||
return "### Prochaines étapes\n\n" + "\n".join(lines)
|
||||
return ""
|
||||
|
||||
|
||||
def post_with_retries(url, data):
|
||||
"""Send POST request with automatic retries."""
|
||||
session = create_retry_session()
|
||||
session.headers.update(
|
||||
{"Authorization": f"Bearer {settings.webhook_api_token.get_secret_value()}"}
|
||||
)
|
||||
try:
|
||||
response = session.post(url, json=data)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
@celery.task(
|
||||
bind=True,
|
||||
autoretry_for=[exceptions.HTTPError],
|
||||
max_retries=settings.celery_max_retries,
|
||||
queue=settings.transcribe_queue,
|
||||
)
|
||||
def process_audio_transcribe_summarize_v2(
|
||||
self,
|
||||
owner_id: str,
|
||||
filename: str,
|
||||
email: str,
|
||||
sub: str,
|
||||
received_at: float,
|
||||
room: Optional[str],
|
||||
recording_date: Optional[str],
|
||||
recording_time: Optional[str],
|
||||
language: Optional[str],
|
||||
download_link: Optional[str],
|
||||
):
|
||||
"""Process an audio file by transcribing it and generating a summary.
|
||||
|
||||
This Celery task performs the following operations:
|
||||
1. Retrieves the audio file from MinIO storage
|
||||
2. Transcribes the audio using WhisperX model
|
||||
3. Sends the results via webhook
|
||||
|
||||
"""
|
||||
logger.info(
|
||||
"Notification received | Owner: %s | Room: %s",
|
||||
owner_id,
|
||||
room,
|
||||
)
|
||||
|
||||
task_id = self.request.id
|
||||
|
||||
logger.info("Initiating WhisperX client")
|
||||
whisperx_client = openai.OpenAI(
|
||||
api_key=settings.whisperx_api_key.get_secret_value(),
|
||||
@@ -73,9 +145,10 @@ def transcribe_audio(task_id, filename, language):
|
||||
max_retries=settings.whisperx_max_retries,
|
||||
)
|
||||
|
||||
# Transcription
|
||||
try:
|
||||
with file_service.prepare_audio_file(filename) as (audio_file, metadata):
|
||||
with (
|
||||
file_service.prepare_audio_file(filename) as (audio_file, metadata),
|
||||
):
|
||||
metadata_manager.track(task_id, {"audio_length": metadata["duration"]})
|
||||
|
||||
if language is None:
|
||||
@@ -106,32 +179,13 @@ def transcribe_audio(task_id, filename, language):
|
||||
|
||||
except FileServiceException:
|
||||
logger.exception("Unexpected error for filename: %s", filename)
|
||||
return None
|
||||
return
|
||||
|
||||
metadata_manager.track_transcription_metadata(task_id, transcription)
|
||||
return transcription
|
||||
|
||||
formatter = TranscriptFormatter()
|
||||
|
||||
def format_transcript(
|
||||
transcription,
|
||||
context_language,
|
||||
language,
|
||||
room,
|
||||
recording_date,
|
||||
recording_time,
|
||||
download_link,
|
||||
):
|
||||
"""Format a transcription into readable content with a title.
|
||||
|
||||
Resolves the locale from context_language / language, then uses
|
||||
TranscriptFormatter to produce markdown content and a title.
|
||||
|
||||
Returns a (content, title) tuple.
|
||||
"""
|
||||
locale = get_locale(context_language, language)
|
||||
formatter = TranscriptFormatter(locale)
|
||||
|
||||
return formatter.format(
|
||||
content, title = formatter.format(
|
||||
transcription,
|
||||
room=room,
|
||||
recording_date=recording_date,
|
||||
@@ -139,93 +193,34 @@ def format_transcript(
|
||||
download_link=download_link,
|
||||
)
|
||||
|
||||
data = {
|
||||
"title": title,
|
||||
"content": content,
|
||||
"email": email,
|
||||
"sub": sub,
|
||||
}
|
||||
|
||||
def format_actions(llm_output: dict) -> str:
|
||||
"""Format the actions from the LLM output into a markdown list.
|
||||
logger.debug("Submitting webhook to %s", settings.webhook_url)
|
||||
logger.debug("Request payload: %s", json.dumps(data, indent=2))
|
||||
|
||||
fomat:
|
||||
- [ ] Action title Assignée à : assignee1, assignee2, Échéance : due_date
|
||||
"""
|
||||
lines = []
|
||||
for action in llm_output.get("actions", []):
|
||||
title = action.get("title", "").strip()
|
||||
assignees = ", ".join(action.get("assignees", [])) or "-"
|
||||
due_date = action.get("due_date") or "-"
|
||||
line = f"- [ ] {title} Assignée à : {assignees}, Échéance : {due_date}"
|
||||
lines.append(line)
|
||||
if lines:
|
||||
return "### Prochaines étapes\n\n" + "\n".join(lines)
|
||||
return ""
|
||||
response = post_with_retries(settings.webhook_url, data)
|
||||
|
||||
try:
|
||||
response_data = response.json()
|
||||
document_id = response_data.get("id", "N/A")
|
||||
except (json.JSONDecodeError, AttributeError):
|
||||
document_id = "Unable to parse response"
|
||||
response_data = response.text
|
||||
|
||||
@celery.task(
|
||||
bind=True,
|
||||
autoretry_for=[exceptions.HTTPError],
|
||||
max_retries=settings.celery_max_retries,
|
||||
queue=settings.transcribe_queue,
|
||||
)
|
||||
def process_audio_transcribe_summarize_v2(
|
||||
self,
|
||||
owner_id: str,
|
||||
filename: str,
|
||||
email: str,
|
||||
sub: str,
|
||||
received_at: float,
|
||||
room: Optional[str],
|
||||
recording_date: Optional[str],
|
||||
recording_time: Optional[str],
|
||||
language: Optional[str],
|
||||
download_link: Optional[str],
|
||||
context_language: Optional[str] = None,
|
||||
):
|
||||
"""Process an audio file by transcribing it and generating a summary.
|
||||
|
||||
This Celery task orchestrates:
|
||||
1. Audio transcription via WhisperX
|
||||
2. Transcript formatting
|
||||
3. Webhook submission
|
||||
4. Conditional summarization queuing
|
||||
|
||||
Args:
|
||||
self: Celery task instance (passed on with bind=True)
|
||||
owner_id: Unique identifier of the recording owner.
|
||||
filename: Name of the audio file in MinIO storage.
|
||||
email: Email address of the recording owner.
|
||||
sub: OIDC subject identifier of the recording owner.
|
||||
received_at: Unix timestamp when the recording was received.
|
||||
room: room name where the recording took place.
|
||||
recording_date: Date of the recording (localized display string).
|
||||
recording_time: Time of the recording (localized display string).
|
||||
language: ISO 639-1 language code for transcription.
|
||||
download_link: URL to download the original recording.
|
||||
context_language: ISO 639-1 language code of the meeting summary context text.
|
||||
"""
|
||||
logger.info(
|
||||
"Notification received | Owner: %s | Room: %s",
|
||||
owner_id,
|
||||
room,
|
||||
"Webhook success | Document %s submitted (HTTP %s)",
|
||||
document_id,
|
||||
response.status_code,
|
||||
)
|
||||
logger.debug("Full response: %s", response_data)
|
||||
|
||||
task_id = self.request.id
|
||||
|
||||
transcription = transcribe_audio(task_id, filename, language)
|
||||
if transcription is None:
|
||||
return
|
||||
|
||||
content, title = format_transcript(
|
||||
transcription,
|
||||
context_language,
|
||||
language,
|
||||
room,
|
||||
recording_date,
|
||||
recording_time,
|
||||
download_link,
|
||||
)
|
||||
|
||||
submit_content(content, title, email, sub)
|
||||
metadata_manager.capture(task_id, settings.posthog_event_success)
|
||||
|
||||
# LLM Summarization
|
||||
if (
|
||||
analytics.is_feature_enabled("summary-enabled", distinct_id=owner_id)
|
||||
and settings.is_summary_enabled
|
||||
@@ -291,11 +286,12 @@ def summarize_transcription(
|
||||
# a singleton client. This is a performance trade-off we accept to ensure per-user
|
||||
# privacy controls in observability traces.
|
||||
llm_observability = LLMObservability(
|
||||
logger=logger,
|
||||
user_has_tracing_consent=user_has_tracing_consent,
|
||||
session_id=self.request.id,
|
||||
user_id=owner_id,
|
||||
)
|
||||
llm_service = LLMService(llm_observability=llm_observability)
|
||||
llm_service = LLMService(llm_observability=llm_observability, logger=logger)
|
||||
|
||||
tldr = llm_service.call(PROMPT_SYSTEM_TLDR, transcript, name="tldr")
|
||||
|
||||
@@ -338,9 +334,22 @@ def summarize_transcription(
|
||||
logger.info("Summary cleaned")
|
||||
|
||||
summary = tldr + "\n\n" + cleaned_summary + "\n\n" + next_steps
|
||||
summary_title = settings.summary_title_template.format(title=title)
|
||||
|
||||
submit_content(summary, summary_title, email, sub)
|
||||
data = {
|
||||
"title": settings.summary_title_template.format(
|
||||
title=title,
|
||||
),
|
||||
"content": summary,
|
||||
"email": email,
|
||||
"sub": sub,
|
||||
}
|
||||
|
||||
logger.debug("Submitting webhook to %s", settings.webhook_url)
|
||||
|
||||
response = post_with_retries(settings.webhook_url, data)
|
||||
|
||||
logger.info("Webhook submitted successfully. Status: %s", response.status_code)
|
||||
logger.debug("Response body: %s", response.text)
|
||||
|
||||
llm_observability.flush()
|
||||
logger.debug("LLM observability flushed")
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""Application configuration and settings."""
|
||||
|
||||
from functools import lru_cache
|
||||
from typing import Annotated, List, Literal, Optional, Set
|
||||
from typing import Annotated, List, Optional, Set
|
||||
|
||||
from fastapi import Depends
|
||||
from pydantic import SecretStr
|
||||
@@ -51,6 +51,7 @@ class Settings(BaseSettings):
|
||||
|
||||
# Transcription processing
|
||||
hallucination_patterns: List[str] = ["Vap'n'Roll Thierry"]
|
||||
hallucination_replacement_text: str = "[Texte impossible à transcrire]"
|
||||
|
||||
# Webhook-related settings
|
||||
webhook_max_retries: int = 2
|
||||
@@ -59,10 +60,11 @@ class Settings(BaseSettings):
|
||||
webhook_api_token: SecretStr
|
||||
webhook_url: str
|
||||
|
||||
# Locale
|
||||
default_context_language: Literal["de", "en", "fr", "nl"] = "fr"
|
||||
|
||||
# Output related settings
|
||||
document_default_title: Optional[str] = "Transcription"
|
||||
document_title_template: Optional[str] = (
|
||||
'Réunion "{room}" du {room_recording_date} à {room_recording_time}'
|
||||
)
|
||||
summary_title_template: Optional[str] = "Résumé de {title}"
|
||||
|
||||
# Summary related settings
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""File service to encapsulate files' manipulations."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -16,9 +15,6 @@ from summary.core.config import get_settings
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FileServiceException(Exception):
|
||||
"""Base exception for file service operations."""
|
||||
|
||||
@@ -28,8 +24,10 @@ class FileServiceException(Exception):
|
||||
class FileService:
|
||||
"""Service for downloading and preparing files from MinIO storage."""
|
||||
|
||||
def __init__(self):
|
||||
def __init__(self, logger):
|
||||
"""Initialize FileService with MinIO client and configuration."""
|
||||
self._logger = logger
|
||||
|
||||
endpoint = (
|
||||
settings.aws_s3_endpoint_url.removeprefix("https://")
|
||||
.removeprefix("http://")
|
||||
@@ -55,16 +53,16 @@ class FileService:
|
||||
The file is downloaded to a temporary location for local manipulation
|
||||
such as validation, conversion, or processing before being used.
|
||||
"""
|
||||
logger.info("Download recording | object_key: %s", remote_object_key)
|
||||
self._logger.info("Download recording | object_key: %s", remote_object_key)
|
||||
|
||||
if not remote_object_key:
|
||||
logger.warning("Invalid object_key '%s'", remote_object_key)
|
||||
self._logger.warning("Invalid object_key '%s'", remote_object_key)
|
||||
raise ValueError("Invalid object_key")
|
||||
|
||||
extension = Path(remote_object_key).suffix.lower()
|
||||
|
||||
if extension not in self._allowed_extensions:
|
||||
logger.warning("Invalid file extension '%s'", extension)
|
||||
self._logger.warning("Invalid file extension '%s'", extension)
|
||||
raise ValueError(f"Invalid file extension '{extension}'")
|
||||
|
||||
response = None
|
||||
@@ -83,8 +81,8 @@ class FileService:
|
||||
tmp.flush()
|
||||
local_path = Path(tmp.name)
|
||||
|
||||
logger.info("Recording successfully downloaded")
|
||||
logger.debug("Recording local file path: %s", local_path)
|
||||
self._logger.info("Recording successfully downloaded")
|
||||
self._logger.debug("Recording local file path: %s", local_path)
|
||||
|
||||
return local_path
|
||||
|
||||
@@ -102,7 +100,7 @@ class FileService:
|
||||
file_metadata = mutagen.File(local_path).info
|
||||
duration = file_metadata.length
|
||||
|
||||
logger.info(
|
||||
self._logger.info(
|
||||
"Recording file duration: %.2f seconds",
|
||||
duration,
|
||||
)
|
||||
@@ -111,14 +109,14 @@ class FileService:
|
||||
error_msg = "Recording too long. Limit is %.2fs seconds" % (
|
||||
self._max_duration,
|
||||
)
|
||||
logger.error(error_msg)
|
||||
self._logger.error(error_msg)
|
||||
raise ValueError(error_msg)
|
||||
|
||||
return duration
|
||||
|
||||
def _extract_audio_from_video(self, video_path: Path) -> Path:
|
||||
"""Extract audio from video file (e.g., MP4) and save as audio file."""
|
||||
logger.info("Extracting audio from video file: %s", video_path)
|
||||
self._logger.info("Extracting audio from video file: %s", video_path)
|
||||
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".m4a", delete=False, prefix="audio_extract_"
|
||||
@@ -142,16 +140,16 @@ class FileService:
|
||||
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True
|
||||
)
|
||||
|
||||
logger.info("Audio successfully extracted to: %s", output_path)
|
||||
self._logger.info("Audio successfully extracted to: %s", output_path)
|
||||
return output_path
|
||||
|
||||
except FileNotFoundError as e:
|
||||
logger.error("ffmpeg not found. Please install ffmpeg.")
|
||||
self._logger.error("ffmpeg not found. Please install ffmpeg.")
|
||||
if output_path.exists():
|
||||
os.remove(output_path)
|
||||
raise RuntimeError("ffmpeg is not installed or not in PATH") from e
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error("Audio extraction failed: %s", e.stderr.decode())
|
||||
self._logger.error("Audio extraction failed: %s", e.stderr.decode())
|
||||
if output_path.exists():
|
||||
os.remove(output_path)
|
||||
raise RuntimeError("Failed to extract audio.") from e
|
||||
@@ -175,7 +173,7 @@ class FileService:
|
||||
extension = downloaded_path.suffix.lower()
|
||||
|
||||
if extension in settings.recording_video_extensions:
|
||||
logger.info("Video file detected, extracting audio...")
|
||||
self._logger.info("Video file detected, extracting audio...")
|
||||
extracted_audio_path = self._extract_audio_from_video(downloaded_path)
|
||||
processed_path = extracted_audio_path
|
||||
else:
|
||||
@@ -196,6 +194,8 @@ class FileService:
|
||||
|
||||
try:
|
||||
os.remove(path)
|
||||
logger.debug("Temporary file removed: %s", path)
|
||||
self._logger.debug("Temporary file removed: %s", path)
|
||||
except OSError as e:
|
||||
logger.warning("Failed to remove temporary file %s: %s", path, e)
|
||||
self._logger.warning(
|
||||
"Failed to remove temporary file %s: %s", path, e
|
||||
)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""LLM service to encapsulate LLM's calls."""
|
||||
|
||||
import logging
|
||||
from typing import Any, Mapping, Optional
|
||||
|
||||
import openai
|
||||
@@ -11,9 +10,6 @@ from summary.core.config import get_settings
|
||||
settings = get_settings()
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LLMObservability:
|
||||
"""Manage observability and tracing for LLM calls using Langfuse.
|
||||
|
||||
@@ -25,11 +21,13 @@ class LLMObservability:
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
logger,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
user_has_tracing_consent: bool = False,
|
||||
):
|
||||
"""Initialize the LLMObservability client."""
|
||||
self._logger = logger
|
||||
self._observability_client: Optional[Langfuse] = None
|
||||
self.session_id = session_id
|
||||
self.user_id = user_id
|
||||
@@ -77,7 +75,7 @@ class LLMObservability:
|
||||
}
|
||||
|
||||
if not self.is_enabled:
|
||||
logger.debug("Using regular OpenAI client (observability disabled)")
|
||||
self._logger.debug("Using regular OpenAI client (observability disabled)")
|
||||
return openai.OpenAI(**base_args)
|
||||
|
||||
# Langfuse's OpenAI wrapper is imported here to avoid triggering client
|
||||
@@ -85,7 +83,7 @@ class LLMObservability:
|
||||
# is missing. Conditional import ensures Langfuse only initializes when enabled.
|
||||
from langfuse.openai import openai as langfuse_openai # noqa: PLC0415
|
||||
|
||||
logger.debug("Using LangfuseOpenAI client (observability enabled)")
|
||||
self._logger.debug("Using LangfuseOpenAI client (observability enabled)")
|
||||
return langfuse_openai.OpenAI(**base_args)
|
||||
|
||||
def flush(self):
|
||||
@@ -101,10 +99,11 @@ class LLMException(Exception):
|
||||
class LLMService:
|
||||
"""Service for performing calls to the LLM configured in the settings."""
|
||||
|
||||
def __init__(self, llm_observability):
|
||||
def __init__(self, llm_observability, logger):
|
||||
"""Init the LLMService once."""
|
||||
self._client = llm_observability.get_openai_client()
|
||||
self._observability = llm_observability
|
||||
self._logger = logger
|
||||
|
||||
def call(
|
||||
self,
|
||||
@@ -141,5 +140,5 @@ class LLMService:
|
||||
return response.choices[0].message.content
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("LLM call failed: %s", e)
|
||||
self._logger.exception("LLM call failed: %s", e)
|
||||
raise LLMException(f"LLM call failed: {e}") from e
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
"""Locale support for the summary service."""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.locales import de, en, fr, nl
|
||||
from summary.core.locales.strings import LocaleStrings
|
||||
|
||||
_LOCALES = {"fr": fr, "en": en, "de": de, "nl": nl}
|
||||
|
||||
|
||||
def get_locale(*languages: Optional[str]) -> LocaleStrings:
|
||||
"""Return locale strings for the first matching language candidate.
|
||||
|
||||
Accept language codes in decreasing priority order and return the
|
||||
locale for the first one that matches a known locale.
|
||||
Fall back to the configured default_context_language.
|
||||
"""
|
||||
for lang in languages:
|
||||
if not lang:
|
||||
continue
|
||||
if lang in _LOCALES:
|
||||
return _LOCALES[lang].STRINGS
|
||||
|
||||
# Provide fallback for longer formats of ISO 639-1 (e.g. "en-au" -> "en")
|
||||
base_lang = lang.split("-")[0]
|
||||
if base_lang in _LOCALES:
|
||||
return _LOCALES[base_lang].STRINGS
|
||||
|
||||
return _LOCALES[get_settings().default_context_language].STRINGS
|
||||
@@ -1,34 +0,0 @@
|
||||
"""German locale strings."""
|
||||
|
||||
from summary.core.locales.strings import LocaleStrings
|
||||
|
||||
STRINGS = LocaleStrings(
|
||||
empty_transcription="""
|
||||
**In Ihrer Transkription wurde kein Audioinhalt erkannt.**
|
||||
|
||||
*Wenn Sie glauben, dass es sich um einen Fehler handelt, zögern Sie nicht,
|
||||
unseren technischen Support zu kontaktieren: visio@numerique.gouv.fr*
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
Einige Punkte, die wir Ihnen empfehlen zu überprüfen:
|
||||
- War ein Mikrofon aktiviert?
|
||||
- Waren Sie nah genug am Mikrofon?
|
||||
- Ist das Mikrofon von guter Qualität?
|
||||
- Dauert die Aufnahme länger als 30 Sekunden?
|
||||
|
||||
""",
|
||||
download_header_template=(
|
||||
"\n*Laden Sie Ihre Aufnahme herunter, "
|
||||
"indem Sie [diesem Link folgen]({download_link})*\n"
|
||||
),
|
||||
hallucination_replacement_text="[Text konnte nicht transkribiert werden]",
|
||||
document_default_title="Transkription",
|
||||
document_title_template=(
|
||||
'Besprechung "{room}" am {room_recording_date} um {room_recording_time}'
|
||||
),
|
||||
)
|
||||
@@ -1,33 +0,0 @@
|
||||
"""English locale strings."""
|
||||
|
||||
from summary.core.locales.strings import LocaleStrings
|
||||
|
||||
STRINGS = LocaleStrings(
|
||||
empty_transcription="""
|
||||
**No audio content was detected in your transcription.**
|
||||
|
||||
*If you believe this is an error, please do not hesitate to contact
|
||||
our technical support: visio@numerique.gouv.fr*
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
A few things we recommend you check:
|
||||
- Was a microphone enabled?
|
||||
- Were you close enough to the microphone?
|
||||
- Is the microphone of good quality?
|
||||
- Is the recording longer than 30 seconds?
|
||||
|
||||
""",
|
||||
download_header_template=(
|
||||
"\n*Download your recording by [following this link]({download_link})*\n"
|
||||
),
|
||||
hallucination_replacement_text="[Unable to transcribe text]",
|
||||
document_default_title="Transcription",
|
||||
document_title_template=(
|
||||
'Meeting "{room}" on {room_recording_date} at {room_recording_time}'
|
||||
),
|
||||
)
|
||||
@@ -1,33 +0,0 @@
|
||||
"""French locale strings (default)."""
|
||||
|
||||
from summary.core.locales.strings import LocaleStrings
|
||||
|
||||
STRINGS = LocaleStrings(
|
||||
empty_transcription="""
|
||||
**Aucun contenu audio n'a été détecté dans votre transcription.**
|
||||
|
||||
*Si vous pensez qu'il s'agit d'une erreur, n'hésitez pas à contacter
|
||||
notre support technique : visio@numerique.gouv.fr*
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
Quelques points que nous vous conseillons de vérifier :
|
||||
- Un micro était-il activé ?
|
||||
- Étiez-vous suffisamment proche ?
|
||||
- Le micro est-il de bonne qualité ?
|
||||
- L'enregistrement dure-t-il plus de 30 secondes ?
|
||||
|
||||
""",
|
||||
download_header_template=(
|
||||
"\n*Télécharger votre enregistrement en [suivant ce lien]({download_link})*\n"
|
||||
),
|
||||
hallucination_replacement_text="[Texte impossible à transcrire]",
|
||||
document_default_title="Transcription",
|
||||
document_title_template=(
|
||||
'Réunion "{room}" du {room_recording_date} à {room_recording_time}'
|
||||
),
|
||||
)
|
||||
@@ -1,33 +0,0 @@
|
||||
"""Dutch locale strings."""
|
||||
|
||||
from summary.core.locales.strings import LocaleStrings
|
||||
|
||||
STRINGS = LocaleStrings(
|
||||
empty_transcription="""
|
||||
**Er is geen audio-inhoud gedetecteerd in uw transcriptie.**
|
||||
|
||||
*Als u denkt dat dit een fout is, aarzel dan niet om contact op te nemen
|
||||
met onze technische ondersteuning: visio@numerique.gouv.fr*
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
Een paar punten die wij u aanraden te controleren:
|
||||
- Was er een microfoon ingeschakeld?
|
||||
- Was u dicht genoeg bij de microfoon?
|
||||
- Is de microfoon van goede kwaliteit?
|
||||
- Duurt de opname langer dan 30 seconden?
|
||||
|
||||
""",
|
||||
download_header_template=(
|
||||
"\n*Download uw opname door [deze link te volgen]({download_link})*\n"
|
||||
),
|
||||
hallucination_replacement_text="[Tekst kon niet worden getranscribeerd]",
|
||||
document_default_title="Transcriptie",
|
||||
document_title_template=(
|
||||
'Vergadering "{room}" op {room_recording_date} om {room_recording_time}'
|
||||
),
|
||||
)
|
||||
@@ -1,15 +0,0 @@
|
||||
"""Locale types for the summary service."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LocaleStrings:
|
||||
"""All translatable output strings for the summary pipeline."""
|
||||
|
||||
# transcript_formatter.py
|
||||
empty_transcription: str
|
||||
download_header_template: str
|
||||
hallucination_replacement_text: str
|
||||
document_default_title: str
|
||||
document_title_template: str
|
||||
@@ -4,13 +4,34 @@ import logging
|
||||
from typing import Optional, Tuple
|
||||
|
||||
from summary.core.config import get_settings
|
||||
from summary.core.locales import LocaleStrings
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
DEFAULT_EMPTY_TRANSCRIPTION = """
|
||||
**Aucun contenu audio n’a été détecté dans votre transcription.**
|
||||
|
||||
|
||||
*Si vous pensez qu’il s’agit d’une erreur, n’hésitez pas à contacter
|
||||
notre support technique : visio@numerique.gouv.fr*
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
.
|
||||
|
||||
Quelques points que nous vous conseillons de vérifier :
|
||||
- Un micro était-il activé ?
|
||||
- Étiez-vous suffisamment proche ?
|
||||
- Le micro est-il de bonne qualité ?
|
||||
- L’enregistrement dure-t-il plus de 30 secondes ?
|
||||
|
||||
"""
|
||||
|
||||
|
||||
class TranscriptFormatter:
|
||||
"""Formats WhisperX transcription output into readable conversation format.
|
||||
|
||||
@@ -21,10 +42,12 @@ class TranscriptFormatter:
|
||||
- Generating descriptive titles from context
|
||||
"""
|
||||
|
||||
def __init__(self, locale: LocaleStrings):
|
||||
"""Initialize formatter with settings and locale."""
|
||||
def __init__(self):
|
||||
"""Initialize formatter with settings."""
|
||||
self.hallucination_patterns = settings.hallucination_patterns
|
||||
self._locale = locale
|
||||
self.hallucination_replacement_text = settings.hallucination_replacement_text
|
||||
self.default_title = settings.document_default_title
|
||||
self.default_empty_transcription = DEFAULT_EMPTY_TRANSCRIPTION
|
||||
|
||||
def _get_segments(self, transcription):
|
||||
"""Extract segments from transcription object or dictionary."""
|
||||
@@ -48,7 +71,7 @@ class TranscriptFormatter:
|
||||
segments = self._get_segments(transcription)
|
||||
|
||||
if not segments:
|
||||
content = self._locale.empty_transcription
|
||||
content = self.default_empty_transcription
|
||||
else:
|
||||
content = self._format_speaker(segments)
|
||||
content = self._remove_hallucinations(content)
|
||||
@@ -60,7 +83,7 @@ class TranscriptFormatter:
|
||||
|
||||
def _remove_hallucinations(self, content: str) -> str:
|
||||
"""Remove hallucination patterns from content."""
|
||||
replacement = self._locale.hallucination_replacement_text or ""
|
||||
replacement = self.hallucination_replacement_text or ""
|
||||
|
||||
for pattern in self.hallucination_patterns:
|
||||
content = content.replace(pattern, replacement)
|
||||
@@ -88,8 +111,9 @@ class TranscriptFormatter:
|
||||
if not download_link:
|
||||
return content
|
||||
|
||||
header = self._locale.download_header_template.format(
|
||||
download_link=download_link
|
||||
header = (
|
||||
f"\n*Télécharger votre enregistrement "
|
||||
f"en [suivant ce lien]({download_link})*\n"
|
||||
)
|
||||
content = header + content
|
||||
|
||||
@@ -103,9 +127,9 @@ class TranscriptFormatter:
|
||||
) -> str:
|
||||
"""Generate title from context or return default."""
|
||||
if not room or not recording_date or not recording_time:
|
||||
return self._locale.document_default_title
|
||||
return self.default_title
|
||||
|
||||
return self._locale.document_title_template.format(
|
||||
return settings.document_title_template.format(
|
||||
room=room,
|
||||
room_recording_date=recording_date,
|
||||
room_recording_time=recording_time,
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
"""Service for delivering content to external destinations."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
|
||||
from requests import Session
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util import Retry
|
||||
|
||||
from summary.core.config import get_settings
|
||||
|
||||
settings = get_settings()
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _create_retry_session():
|
||||
"""Create an HTTP session configured with retry logic."""
|
||||
session = Session()
|
||||
retries = Retry(
|
||||
total=settings.webhook_max_retries,
|
||||
backoff_factor=settings.webhook_backoff_factor,
|
||||
status_forcelist=settings.webhook_status_forcelist,
|
||||
allowed_methods={"POST"},
|
||||
)
|
||||
session.mount("https://", HTTPAdapter(max_retries=retries))
|
||||
return session
|
||||
|
||||
|
||||
def _post_with_retries(url, data):
|
||||
"""Send POST request with automatic retries."""
|
||||
session = _create_retry_session()
|
||||
session.headers.update(
|
||||
{"Authorization": f"Bearer {settings.webhook_api_token.get_secret_value()}"}
|
||||
)
|
||||
try:
|
||||
response = session.post(url, json=data)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def submit_content(content, title, email, sub):
|
||||
"""Submit content to the configured webhook destination.
|
||||
|
||||
Builds the payload, sends it with retries, and logs the outcome.
|
||||
"""
|
||||
data = {
|
||||
"title": title,
|
||||
"content": content,
|
||||
"email": email,
|
||||
"sub": sub,
|
||||
}
|
||||
|
||||
logger.debug("Submitting to %s", settings.webhook_url)
|
||||
logger.debug("Request payload: %s", json.dumps(data, indent=2))
|
||||
|
||||
response = _post_with_retries(settings.webhook_url, data)
|
||||
|
||||
try:
|
||||
response_data = response.json()
|
||||
document_id = response_data.get("id", "N/A")
|
||||
except (json.JSONDecodeError, AttributeError):
|
||||
document_id = "Unable to parse response"
|
||||
response_data = response.text
|
||||
|
||||
logger.info(
|
||||
"Delivery success | Document %s submitted (HTTP %s)",
|
||||
document_id,
|
||||
response.status_code,
|
||||
)
|
||||
logger.debug("Full response: %s", response_data)
|
||||
Reference in New Issue
Block a user