fixup! fixup! 🎉(conversations) bootstrap backend & frontend

This commit needs to be removed before merge.

.codespellrc

@@ -1,16 +0,0 @@
-[codespell]
-skip =
-    ./git/,
-    **/*.po,
-    **/*.pot,
-    **/*.json,
-    **/yarn.lock,
-    **/node_modules/**,
-    **/e2e/report/**,
-    *.tsbuildinfo,
-check-filenames = true
-ignore-words-list =
-    afterAll,
-    statics,
-exclude-file =
-    ./src/backend/chat/agent_rag/web_search/mocked.py,

.github/ISSUE_TEMPLATE/Bug_report.md

@@ -18,7 +18,7 @@ A clear and concise description of what you expected to happen (or code).
 3. And then the bug happens!
 
 **Environment**
-- Conversations version:
+- Docs version:
 - Instance url:
 
 **Possible Solution**

.github/workflows/conversations-frontend.yml

@@ -18,7 +18,7 @@ jobs:
 
   test-front:
     needs: install-dependencies
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -28,9 +28,6 @@ jobs:
         with:
           node-version: "22.x"
 
-      - name: Install yarn
-        run: npm install -g yarn
-
       - name: Restore the frontend cache
         uses: actions/cache@v4
         with:
@@ -42,7 +39,7 @@ jobs:
         run: cd src/frontend/ && yarn test
 
   lint-front:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     needs: install-dependencies
     steps:
       - name: Checkout repository
@@ -52,10 +49,6 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: "22.x"
-
-      - name: Install yarn
-        run: npm install -g yarn
-
       - name: Restore the frontend cache
         uses: actions/cache@v4
         with:
@@ -67,9 +60,9 @@ jobs:
         run: cd src/frontend/ && yarn lint
 
   test-e2e-chromium:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     needs: install-dependencies
-    timeout-minutes: 40
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -79,9 +72,6 @@ jobs:
         with:
           node-version: "22.x"
 
-      - name: Install yarn
-        run: npm install -g yarn
-
       - name: Restore the frontend cache
         uses: actions/cache@v4
         with:
@@ -109,9 +99,9 @@ jobs:
           retention-days: 7
 
   test-e2e-other-browser:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     needs: test-e2e-chromium
-    timeout-minutes: 40
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
@@ -121,9 +111,6 @@ jobs:
         with:
           node-version: "22.x"
 
-      - name: Install yarn
-        run: npm install -g yarn
-
       - name: Restore the frontend cache
         uses: actions/cache@v4
         with:

.github/workflows/conversations.yml

@@ -15,7 +15,7 @@ jobs:
       with-build_mails: true
 
   lint-git:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     if: github.event_name == 'pull_request' # Makes sense only for pull requests
     steps:
       - name: Checkout repository
@@ -25,22 +25,18 @@ jobs:
       - name: show
         run: git log
       - name: Enforce absence of print statements in code
-        if: always()
         run: |
           ! git diff origin/${{ github.event.pull_request.base.ref }}..HEAD -- . ':(exclude)**/conversations.yml' | grep "print("
       - name: Check absence of fixup commits
-        if: always()
         run: |
           ! git log | grep 'fixup!'
       - name: Install gitlint
-        if: always()
         run: pip install --user requests gitlint
       - name: Lint commit messages added to main
-        if: always()
         run: ~/.local/bin/gitlint --commits origin/${{ github.event.pull_request.base.ref }}..HEAD
 
   check-changelog:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     if: |
       contains(github.event.pull_request.labels.*.name, 'noChangeLog') == false &&
       github.event_name == 'pull_request'
@@ -53,7 +49,7 @@ jobs:
         run: git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.event.after }} | grep 'CHANGELOG.md'
 
   lint-changelog:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
@@ -66,7 +62,7 @@ jobs:
           fi
 
   lint-spell-mistakes:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     if: github.event_name == 'pull_request'
     steps:
       - name: Checkout repository
@@ -74,10 +70,18 @@ jobs:
       - name: Install codespell
         run: pip install --user codespell
       - name: Check for typos
-        run: codespell
+        run: |
+          codespell \
+            --check-filenames \
+            --ignore-words-list "Dokument,afterAll,excpt,statics" \
+            --skip "./git/" \
+            --skip "**/*.po" \
+            --skip "**/*.pot" \
+            --skip "**/*.json" \
+            --skip "**/yarn.lock"
 
   lint-back:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     defaults:
       run:
         working-directory: src/backend
@@ -85,10 +89,9 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v2
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v3
         with:
           python-version: "3.13.3"
-          cache: "pip"
       - name: Upgrade pip and setuptools
         run: pip install --upgrade pip setuptools
       - name: Install development dependencies
@@ -101,7 +104,7 @@ jobs:
         run: ~/.local/bin/pylint .
 
   test-back:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     needs: install-dependencies
 
     defaults:
@@ -181,10 +184,9 @@ jobs:
             mc version enable conversations/conversations-media-storage"
 
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v3
         with:
           python-version: "3.13.3"
-          cache: "pip"
 
       - name: Install development dependencies
         run: pip install --user .[dev]

.github/workflows/crowdin_download.yml

@@ -14,7 +14,7 @@ jobs:
       with-front-dependencies-installation: true
 
   synchronize-with-crowdin:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     permissions:
       contents: write
       pull-requests: write
@@ -47,12 +47,6 @@ jobs:
 
           CROWDIN_BASE_PATH: "../src/"
       # frontend i18n
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: "22.x"
-      - name: Install yarn
-        run: npm install -g yarn
       - name: Restore the frontend cache
         uses: actions/cache@v4
         with:
@@ -80,4 +74,4 @@ jobs:
 
             - [x]  update translated strings
           branch: i18n/update-translations
-          labels: i18n,noChangeLog
+          labels: i18n

.github/workflows/crowdin_upload.yml

@@ -16,17 +16,16 @@ jobs:
 
   synchronize-with-crowdin:
     needs: install-dependencies
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
 
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       # Backend i18n
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v3
         with:
           python-version: "3.13.3"
-          cache: "pip"
       - name: Upgrade pip and setuptools
         run: pip install --upgrade pip setuptools
       - name: Install development dependencies
@@ -48,12 +47,6 @@ jobs:
         run: |
           DJANGO_CONFIGURATION=Build python manage.py makemessages -a --keep-pot
       # frontend i18n
-      - name: Setup Node.js
-        uses: actions/setup-node@v4
-        with:
-          node-version: "22.x"
-      - name: Install yarn
-        run: npm install -g yarn
       - name: Restore the frontend cache
         uses: actions/cache@v4
         with:

.github/workflows/dependencies.yml

@@ -17,32 +17,24 @@ on:
 jobs:
   front-dependencies-installation:
     if: ${{ inputs.with-front-dependencies-installation == true }}
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-
       - name: Restore the frontend cache
         uses: actions/cache@v4
         id: front-node_modules
         with:
           path: "src/frontend/**/node_modules"
           key: front-node_modules-${{ hashFiles('src/frontend/**/yarn.lock') }}
-
       - name: Setup Node.js
         if: steps.front-node_modules.outputs.cache-hit != 'true'
         uses: actions/setup-node@v4
         with:
           node-version: ${{ inputs.node_version }}
-
-      - name: Install yarn
-        if: steps.front-node_modules.outputs.cache-hit != 'true'
-        run: npm install -g yarn
-
       - name: Install dependencies
         if: steps.front-node_modules.outputs.cache-hit != 'true'
         run: cd src/frontend/ && yarn install --frozen-lockfile
-
       - name: Cache install frontend
         if: steps.front-node_modules.outputs.cache-hit != 'true'
         uses: actions/cache@v4
@@ -52,7 +44,7 @@ jobs:
 
   build-mails:
     if: ${{ inputs.with-build_mails == true }}
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     defaults:
       run:
         working-directory: src/mail

.github/workflows/docker-hub.yml

@@ -18,14 +18,11 @@ env:
 
 jobs:
   build-and-push-backend:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       -
         name: Checkout repository
         uses: actions/checkout@v4
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
       -
         name: Docker meta
         id: meta
@@ -35,10 +32,7 @@ jobs:
       -
         name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+        run: echo "${{ secrets.DOCKER_HUB_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_HUB_USER }}" --password-stdin
       -
         name: Run trivy scan
         uses: numerique-gouv/action-trivy-cache@main
@@ -57,14 +51,11 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
 
   build-and-push-frontend:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       -
         name: Checkout repository
         uses: actions/checkout@v4
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
       -
         name: Docker meta
         id: meta
@@ -74,10 +65,7 @@ jobs:
       -
         name: Login to DockerHub
         if: github.event_name != 'pull_request'
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_HUB_USER }}
-          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+        run: echo "${{ secrets.DOCKER_HUB_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_HUB_USER }}" --password-stdin
       -
         name: Run trivy scan
         uses: numerique-gouv/action-trivy-cache@main
@@ -101,7 +89,7 @@ jobs:
     needs:
       - build-and-push-frontend
       - build-and-push-backend
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     if: github.event_name != 'pull_request'
     steps:
       - uses: numerique-gouv/action-argocd-webhook-notification@main

.github/workflows/helmfile-linter.yaml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   helmfile-lint:
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     container:
       image: ghcr.io/helmfile/helmfile:v0.171.0
     steps:
@@ -21,10 +21,10 @@ jobs:
       shell: bash
       run: |
         set -e
-        HELMFILE=src/helm/helmfile.yaml.gotmpl
+        HELMFILE=src/helm/helmfile.yaml
         environments=$(awk 'BEGIN {in_env=0} /^environments:/ {in_env=1; next} /^---/ {in_env=0} in_env && /^  [^ ]/ {gsub(/^  /,""); gsub(/:.*$/,""); print}' "$HELMFILE")
         for env in $environments; do
           echo "################### $env lint ###################"
           helmfile -e $env -f $HELMFILE lint || exit 1
           echo -e "\n"
-        done
+        done

.github/workflows/release-helm-chart.yaml

@@ -12,7 +12,7 @@ jobs:
     # see: https://docs.github.com/en/actions/security-guides/automatic-token-authentication#modifying-the-permissions-for-the-github_token
     permissions:
       contents: write
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.gitignore

@@ -76,6 +76,3 @@ db.sqlite3
 .vscode/
 *.iml
 .devcontainer
-
-# Docker compose override
-compose.override.yml

CHANGELOG.md

@@ -8,80 +8,5 @@ and this project adheres to
 
 ## [Unreleased]
 
-## [0.0.2] - 2025-10-21
 
-### Added
-
-- ✨(front) add drag'n drop file
-- ✨(activation-codes) register users also on Brevo #98
-- 📈(posthog) add `sub` field to tracking #95
-
-### Changed
-- 🔧(front) change links feedback tchap + settings popup
-- 🐛(front) code activation fix session end #93
-- 💬(wording) error page wording #102
-- ⚡️(web-search) allow to override returned chunks #107
-- 🐛(activation-codes) create contact in brevo before add to list #108
-- ⚗️(summarization) add system prompt to handle tool #112
-
-
-## [0.0.1] - 2025-10-19
-
-### Changed
-
-- 🎨(front) activation page footer
-- 👷(front) change size small modal
-- 🎨(front) retour ui global
-- 👷(front) fix button scrollDown
-- 💥(front) disable input when error occurred
-- 👷(front) fix scroll
-- 🐛(front) fix left panel status + fix scroll
-- 🐛(llm) add is_active field and persist chat preference
-- ✨(frontend) add LLM selection in chat input #53
-- 🎨(front) fix width chat container
-- 🎨(front) fix width chat container #55
-- 🐛(front) fix button search web on new conversation
-- 🎨(front) improvement search input scroll
-- ✨(404) fix front 404 page
-- ✅(chat) add frontend feature flags #29
-- 🎨(front) change list attachment in chat
-- 🎨(front) move emplacement for attachment
-- 🎨(ui) retour ui sources files
-- ✨(ui) fix retour global ui 
-- 🐛(fix) broken staging css
-- 🎨(alpha) adjustment for alpha version
-- ✨(ui) delete flex message
-- ✅(front) add enabled/disabled conversation analysis
-- 🎨(front) amelioration chat ux
-- 🎨(front) global layout modification
-- ✨(front) global layout UI
-- ♻️(chat) rewrite backend using Pydantic AI SDK #4
-- 🗃️(chat) enforce messages stored JSON format #6
-- 🐛(chat) UI messages must have a unique identifier #6
-- ✨(llm) allow configuration from JSON file #22
-- 💥(agent) replace routing w/ tool calls #40
-- 🧱(storage) upload the user documents into S3 #86
-
-### Added
-
-- 🎉(conversations) bootstrap backend & frontend #1
-- ✨(web-search) add RAG capability to do web search #7
-- ✨(chat) add document RAG on document uploaded by user #8
-- ✨(backend) allow use to stop conversation streaming #14
-- 🐛(agent) add the current date in the system prompt #18
-- ✨(backend) add feature flags from posthog #13
-- ✨(user) allow to use conversation data for analytics #23
-- ✨(chat) enforce response in user language #24
-- 📈(langfuse) add light instrumentation #26
-- 🚑️(agent) allow Mistral w/ vLLM & tools #36
-- ✨(web-search) add Brave search tool #47
-- ✨(models) add mistral support & customization #51
-- 🐛(web-search) add summarization to Brave results #58
-- ✨(langfuse) allow user to score messages from LLM #6
-- ✨(onboarding) add activation code logic for launch #62
-- 💄(chat) add code highlighting for LLM responses #67
-
-
-[unreleased]: https://github.com/suitenumerique/conversations/compare/v0.0.2...main
-[0.0.2]: https://github.com/suitenumerique/conversations/releases/v0.0.2
-[0.0.1]: https://github.com/suitenumerique/conversations/releases/v0.0.1
+[unreleased]: https://github.com/numerique-gouv/conversations/compare/HEAD...main

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 Thank you for taking the time to contribute! Please follow these guidelines to ensure a smooth and productive workflow. 🚀🚀🚀
 
-To get started with the project, please refer to the [README.md](https://github.com/suitenumerique/conversations/blob/main/README.md) for detailed instructions on how to run Conversations locally.
+To get started with the project, please refer to the [README.md](https://github.com/suitenumerique/conversations/blob/main/README.md) for detailed instructions on how to run Docs locally.
 
 Contributors are required to sign off their commits with `git commit --signoff`: this confirms that they have read and accepted the [Developer's Certificate of Origin 1.1](https://developercertificate.org/). For security reasons we also require [signing your commits with your SSH or GPG key](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification) with `git commit -S`.
 
@@ -92,3 +92,11 @@ Make sure that all new features or fixes have corresponding tests. Run the test
 If you need any help while contributing, feel free to open a discussion or ask for guidance in the issue tracker. We are more than happy to assist!
 
 Thank you for your contributions! 👍
+
+## Contribute to BlockNote
+We use [BlockNote](https://www.blocknotejs.org/) for the text editing features of Docs. 
+If you find and issue with the editor you can [report it](https://github.com/TypeCellOS/BlockNote/issues) directly on their repository.
+
+Please consider contributing to BlockNotejs, as a library, it's useful to many projects not just Docs.
+
+The project is licended with Mozilla Public License Version 2.0 but be aware that [XL packages](https://github.com/TypeCellOS/BlockNote/blob/main/packages/xl-docx-exporter/LICENSE) are dual licenced with GNU AFFERO GENERAL PUBLIC LICENCE Version 3 and proprietary licence if you are [sponsor](https://www.blocknotejs.org/pricing).

Makefile

@@ -138,7 +138,7 @@ logs: ## display app-dev logs (follow mode)
 .PHONY: logs
 
 run-backend: ## Start only the backend application and all needed services
-	@$(COMPOSE) up --force-recreate -d nginx app-dev
+	@$(COMPOSE) up --force-recreate -d nginx ml-flow app-dev
 .PHONY: run-backend
 
 run-frontend: ## Start only the frontend application
@@ -151,10 +151,6 @@ run:
 	@$(MAKE) run-frontend
 .PHONY: run
 
-create-compose-with-models: ## override the docker-compose file with models
-	cp -n compose.with_model.override.yml compose.override.yml
-.PHONY: create-compose-with-models
-
 run-e2e: ## start the e2e server
 run-e2e:
 	@$(MAKE) run-backend

README.md

@@ -16,24 +16,6 @@
 </p>
 
 
-**Warning:** This project is in active development and in a very early stage. Breaking changes may occur at any time.
-
-
-## Yet another AI chatbot
-
-Conversations is an open-source AI chatbot designed to be simple, secure and privacy-friendly.
-
-Why another AI chatbot? Because we want to be able to fully control our data and the way we interact with AI.
-We want to have a very friendly end-user interface and code, and we want to be able to easily customize the
-chatbot to our needs.
-
-We leverage open-source projects such as [Vercel&lsquo;s AI SDK](https://ai-sdk.dev/) and [Pydantic AI](https://ai.pydantic.dev)
-and only assemble them in a way that makes sense for us and allows us to focus on the product.
-
-This assistant's purpose is also to be integrated into the "La Suite numérique" ecosystem of tools for public services.
-
-Any help to improve the project is very welcome!
-
 
 ### Self-host
 🚀 Conversations is easy to install on your own servers
@@ -48,9 +30,9 @@ In the works: Docker Compose, soon YunoHost
 
 You can test Conversations on your browser by visiting this => TBD
 
-### Run Conversations locally
+### Run Docs locally
 
-> ⚠️ The methods described below for running Conversations locally is **for testing purposes only**.
+> ⚠️ The methods described below for running Docs locally is **for testing purposes only**. It is based on building Docs using [Minio](https://min.io/) as an S3-compatible storage solution. Of course you can choose any S3-compatible storage solution.
 
 **Prerequisite**
 
@@ -170,7 +152,9 @@ docs
 
 ### Stack
 
-Conversations is built on top of [Django Rest Framework](https://www.django-rest-framework.org/), [Next.js](https://nextjs.org/), [Vercel&lsquo;s AI SDK](https://ai-sdk.dev/) and [Pydantic AI](https://ai.pydantic.dev). We thank the contributors of all these projects for their awesome work!
+Conversations is built on top of [Django Rest Framework](https://www.django-rest-framework.org/), [Next.js](https://nextjs.org/), [Vercel&lsquo;s AI SDK](https://ai-sdk.dev/) and [OpenAI Agents SDK](https://github.com/openai/openai-agents-python). We thank the contributors of all these projects for their awesome work!
+
+
 
 
 ### Gov ❤️ open source

SECURITY.md

@@ -6,7 +6,7 @@ Security is very important to us.
 
 If you have any issue regarding security, please disclose the information responsibly submitting [this form](https://vdp.numerique.gouv.fr/p/Send-a-report?lang=en) and not by creating an issue on the repository. You can also email us at docs@numerique.gouv.fr
 
-We appreciate your effort to make Conversations more secure.
+We appreciate your effort to make Docs more secure.
 
 ## Vulnerability disclosure policy
 
@@ -20,4 +20,4 @@ and Exposures (CVE) identifier for the vulnerability.
 3. Once this grace period has passed, we will publish the vulnerability.
 
 By adhering to this security policy, we aim to address security concerns
-effectively and responsibly in our open source software project.
+effectively and responsibly in our open source software project.

bin/Tiltfile

@@ -10,6 +10,10 @@ docker_build(
     target = 'backend-production',
     live_update=[
         sync('../src/backend', '/app'),
+        run(
+            'pip install -r /app/requirements.txt',
+            trigger=['./api/requirements.txt']
+        )
     ]
 )
 
@@ -24,9 +28,9 @@ docker_build(
     ]
 )
 
-k8s_resource('conversations-backend-migrate', resource_deps=['postgres-postgresql'])
-k8s_resource('conversations-backend-createsuperuser', resource_deps=['conversations-backend-migrate'])
-k8s_resource('conversations-backend', resource_deps=['conversations-backend-migrate'])
+k8s_resource('conversations-conversations-backend-migrate', resource_deps=['postgres-postgresql'])
+k8s_resource('conversations-conversations-backend-createsuperuser', resource_deps=['conversations-conversations-backend-migrate'])
+k8s_resource('conversations-conversations-backend', resource_deps=['conversations-conversations-backend-migrate'])
 k8s_yaml(local('cd ../src/helm && helmfile -n conversations -e dev template .'))
 
 migration = '''

compose.with_model.override.yml

@@ -1,12 +0,0 @@
-name: conversations
-
-services:
-  app-dev:
-    models:
-      llm:
-        endpoint_var: AI_BASE_URL
-        model_var: AI_MODEL
-
-models:
-  llm:
-    model: ai/smollm2

compose.yml

@@ -84,6 +84,8 @@ services:
           condition: service_started
         createbuckets:
           condition: service_started
+        ai_runner:
+          condition: service_started
 
   nginx:
     image: nginx:1.25
@@ -177,3 +179,20 @@ services:
       kc_postgresql:
         condition: service_healthy
         restart: true
+
+  ml-flow:
+    image: ghcr.io/mlflow/mlflow:v3.0.0
+    environment:
+      MLFLOW_TRACKING_URI: http://localhost:5050
+      MLFLOW_ARTIFACT_ROOT: /mlflow/artifacts
+    ports:
+      - "5050:5050"
+    volumes:
+      - ./data/mlflow:/mlflow/artifacts
+    command: mlflow server --host 0.0.0.0:5050
+
+  ai_runner:
+    provider:
+      type: model
+      options:
+        model: ai/smollm2

docker/files/etc/nginx/conf.d/default.conf

@@ -25,7 +25,7 @@ server {
     }
 
     location /media-auth {
-        proxy_pass http://app-dev:8000/api/v1.0/chats/media-auth/;
+        proxy_pass http://app-dev:8000/api/v1.0/documents/media-auth/;
         proxy_set_header Host $host;
         proxy_set_header X-Real-IP $remote_addr;
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;

docs/architecture.md

@@ -6,9 +6,14 @@
 flowchart TD
     User -- HTTP --> Front("Frontend (NextJS SPA)")
     Front -- REST API --> Back("Backend (Django)")
+    Front -- WebSocket --> Yserver("Microservice Yjs (Express)") -- WebSocket -->  CollaborationServer("Collaboration server (Hocuspocus)") -- REST API <--> Back
     Front -- OIDC --> Back -- OIDC ---> OIDC("Keycloak / ProConnect")
     Back -- REST API --> Yserver
     Back --> DB("Database (PostgreSQL)")
     Back <--> Celery --> DB
     Back ----> S3("Minio (S3)")
 ```
+
+### Architecture decision records
+
+- [ADR-0001-20250106-use-yjs-for-docs-editing](./adr/ADR-0001-20250106-use-yjs-for-docs-editing.md)

docs/env.md

@@ -24,7 +24,7 @@ These are the environment variables you can set for the `conversations-backend`
 | AWS_S3_SECRET_ACCESS_KEY                        | access key for s3 endpoint                                                                                                        |                                                         |
 | AWS_S3_REGION_NAME                              | region name for s3 endpoint                                                                                                       |                                                         |
 | AWS_STORAGE_BUCKET_NAME                         | bucket name for s3 endpoint                                                                                                       | conversations-media-storage                             |
-| ATTACHMENT_MAX_SIZE                         | maximum size of document in bytes                                                                                                 | 10485760                                                |
+| DOCUMENT_IMAGE_MAX_SIZE                         | maximum size of document in bytes                                                                                                 | 10485760                                                |
 | LANGUAGE_CODE                                   | default language                                                                                                                  | en-us                                                   |
 | API_USERS_LIST_THROTTLE_RATE_SUSTAINED          | throttle rate for api                                                                                                             | 180/hour                                                |
 | API_USERS_LIST_THROTTLE_RATE_BURST              | throttle rate for api on burst                                                                                                    | 30/minute                                               |
@@ -79,6 +79,7 @@ These are the environment variables you can set for the `conversations-backend`
 | AI_API_KEY                                      | AI key to be used for AI Base url                                                                                                 |                                                         |
 | AI_BASE_URL                                     | OpenAI compatible AI base url                                                                                                     |                                                         |
 | AI_MODEL                                        | AI Model to use                                                                                                                   |                                                         |
+| AI_AGENT_NAME                                   | Name of the AI agent (useless)                                                                                                    | Conversations Assistant                                 |
 | AI_AGENT_INSTRUCTION                            | Base instruction for the AI agent                                                                                                 | You are a helpful assistant                             |
 | Y_PROVIDER_API_KEY                              | Y provider API key                                                                                                                |                                                         |
 | Y_PROVIDER_API_BASE_URL                         | Y Provider url                                                                                                                    |                                                         |

docs/examples/conversations.values.yaml

@@ -123,7 +123,7 @@ ingressMedia:
   host: conversations.127.0.0.1.nip.io
 
   annotations:
-    nginx.ingress.kubernetes.io/auth-url: https://conversations.127.0.0.1.nip.io/api/v1.0/chats/media-auth/
+    nginx.ingress.kubernetes.io/auth-url: https://conversations.127.0.0.1.nip.io/api/v1.0/documents/media-auth/
     nginx.ingress.kubernetes.io/auth-response-headers: "Authorization, X-Amz-Date, X-Amz-Content-SHA256"
     nginx.ingress.kubernetes.io/upstream-vhost: minio.conversations.svc.cluster.local:9000
     nginx.ingress.kubernetes.io/rewrite-target: /conversations-media-storage/$1

docs/system-requirements.md

@@ -1,9 +1,9 @@
-# La Suite Conversations – System & Requirements (2025-06)
+# La Suite Docs – System & Requirements (2025-06)
 
 ## 1. Quick-Reference Matrix (single VM / laptop)
 
 | Scenario                  | RAM   | vCPU | SSD     | Notes                     |
-|---------------------------|-------|------|---------|---------------------------|
+| ------------------------- | ----- | ---- | ------- | ------------------------- |
 | **Solo dev**              | 8 GB  | 4    | 15 GB   | Hot-reload + one IDE      |
 | **Team QA**               | 16 GB | 6    | 30 GB   | Runs integration tests    |
 | **Prod ≤ 100 live users** | 32 GB | 8 +  | 50 GB + | Scale linearly above this |
@@ -14,20 +14,22 @@ Memory is the first bottleneck; CPU matters only when Celery or the Next.js buil
 
 ## 2. Development Environment Memory Requirements
 
-| Service               | Typical use                   | Rationale / source                                                                      |
-|-----------------------|-------------------------------|-----------------------------------------------------------------------------------------|
-| PostgreSQL            | **1 – 2 GB**                  | `shared_buffers` starting point ≈ 25% RAM ([postgresql.org][1])                         |
-| Keycloak              | **≈ 1.3 GB**                  | 70% of limit for heap + ~300 MB non-heap ([keycloak.org][2])                            |
-| Redis                 | **≤ 256 MB**                  | Empty instance ≈ 3 MB; budget 256 MB to allow small datasets ([stackoverflow.com][3])   |
-| MinIO                 | **2 GB (dev) / 32 GB (prod)** | Pre-allocates 1–2 GiB; docs recommend 32 GB per host for ≤ 100 Ti storage ([min.io][4]) |
-| Django API (+ Celery) | **0.8 – 1.5 GB**              | Empirical in-house metrics                                                              |
-| Next.js frontend      | **0.5 – 1 GB**                | Dev build chain                                                                         |
-| Nginx                 | **< 100 MB**                  | Static reverse-proxy footprint                                                          |
+| Service                  | Typical use                   | Rationale / source                                                                      |
+| ------------------------ | ----------------------------- | --------------------------------------------------------------------------------------- |
+| PostgreSQL               | **1 – 2 GB**                 | `shared_buffers` starting point ≈ 25% RAM ([postgresql.org][1])                       |
+| Keycloak                 | **≈ 1.3 GB**                 | 70% of limit for heap + ~300 MB non-heap ([keycloak.org][2])                         |
+| Redis                    | **≤ 256 MB**                 | Empty instance ≈ 3 MB; budget 256 MB to allow small datasets ([stackoverflow.com][3]) |
+| MinIO                    | **2 GB (dev) / 32 GB (prod)**| Pre-allocates 1–2 GiB; docs recommend 32 GB per host for ≤ 100 Ti storage ([min.io][4]) |
+| Django API (+ Celery)    | **0.8 – 1.5 GB**              | Empirical in-house metrics                                                              |
+| Next.js frontend         | **0.5 – 1 GB**                | Dev build chain                                                                         |
+| Y-Provider (y-websocket) | **< 200 MB**                  | Large 40 MB YDoc called “big” in community thread ([discuss.yjs.dev][5])                |
+| Nginx                    | **< 100 MB**                  | Static reverse-proxy footprint                                                          |
 
 [1]: https://www.postgresql.org/docs/9.1/runtime-config-resource.html "PostgreSQL: Documentation: 9.1: Resource Consumption"
 [2]: https://www.keycloak.org/high-availability/concepts-memory-and-cpu-sizing "Concepts for sizing CPU and memory resources - Keycloak"
 [3]: https://stackoverflow.com/questions/45233052/memory-footprint-for-redis-empty-instance "Memory footprint for Redis empty instance - Stack Overflow"
 [4]: https://min.io/docs/minio/kubernetes/upstream/operations/checklists/hardware.html "Hardware Checklist — MinIO Object Storage for Kubernetes"
+[5]: https://discuss.yjs.dev/t/understanding-memory-requirements-for-production-usage/198 "Understanding memory requirements for production usage - Yjs Community"
 
 > **Rule of thumb:** add 2 GB for OS/overhead, then sum only the rows you actually run.
 
@@ -35,15 +37,16 @@ Memory is the first bottleneck; CPU matters only when Celery or the Next.js buil
 
 Production deployments differ significantly from development environments. The table below shows typical memory usage for production services:
 
-| Service                   | Typical use                 | Rationale / notes                                                                        |
-|---------------------------|-----------------------------|------------------------------------------------------------------------------------------|
-| PostgreSQL                | **2 – 8 GB**                | Higher `shared_buffers` and connection pooling for concurrent users                      |
-| OIDC Provider (optional)  | **Variable**                | Any OIDC-compatible provider (Keycloak, Auth0, Azure AD, etc.) - external or self-hosted |
-| Redis                     | **256 MB – 2 GB**           | Session storage and caching; scales with active user sessions                            |
-| Object Storage (optional) | **External or self-hosted** | Can use AWS S3, Azure Blob, Google Cloud Storage, or self-hosted MinIO                   |
-| Django API (+ Celery)     | **1 – 3 GB**                | Production workloads with background tasks and higher concurrency                        |
-| Static Files (Nginx)      | **< 200 MB**                | Serves Next.js build output and static assets; no development overhead                   |
-| Nginx (Load Balancer)     | **< 200 MB**                | Reverse proxy, SSL termination, static file serving                                      |
+| Service                  | Typical use                   | Rationale / notes                                                                       |
+| ------------------------ | ----------------------------- | --------------------------------------------------------------------------------------- |
+| PostgreSQL               | **2 – 8 GB**                 | Higher `shared_buffers` and connection pooling for concurrent users                    |
+| OIDC Provider (optional) | **Variable**                  | Any OIDC-compatible provider (Keycloak, Auth0, Azure AD, etc.) - external or self-hosted |
+| Redis                    | **256 MB – 2 GB**             | Session storage and caching; scales with active user sessions                          |
+| Object Storage (optional)| **External or self-hosted**   | Can use AWS S3, Azure Blob, Google Cloud Storage, or self-hosted MinIO               |
+| Django API (+ Celery)    | **1 – 3 GB**                 | Production workloads with background tasks and higher concurrency                      |
+| Static Files (Nginx)     | **< 200 MB**                 | Serves Next.js build output and static assets; no development overhead                |
+| Y-Provider (y-websocket) | **200 MB – 1 GB**             | Scales with concurrent document editing sessions                                        |
+| Nginx (Load Balancer)    | **< 200 MB**                  | Reverse proxy, SSL termination, static file serving                                    |
 
 ### Production Architecture Notes
 
@@ -51,7 +54,7 @@ Production deployments differ significantly from development environments. The t
 - **Authentication**: Any OIDC-compatible provider can be used instead of self-hosted Keycloak
 - **Object Storage**: External services (S3, Azure Blob) or self-hosted solutions (MinIO) are both viable
 - **Database**: Consider PostgreSQL clustering or managed database services for high availability
-- **Scaling**: Horizontal scaling is recommended for Django API service
+- **Scaling**: Horizontal scaling is recommended for Django API and Y-Provider services
 
 ### Minimal Production Setup (Core Services Only)
 
@@ -66,7 +69,7 @@ Production deployments differ significantly from development environments. The t
 ## 4. Recommended Software Versions
 
 | Tool                    | Minimum |
-|-------------------------|---------|
+| ----------------------- | ------- |
 | Docker Engine / Desktop | 24.0    |
 | Docker Compose          | v2      |
 | Git                     | 2.40    |
@@ -82,9 +85,10 @@ Production deployments differ significantly from development environments. The t
 ## 5. Ports (dev defaults)
 
 | Port      | Service               |
-|-----------|-----------------------|
+| --------- |-----------------------|
 | 3000      | Next.js               |
 | 8071      | Django                |
+| 4444      | Y-Provider            |
 | 8080      | Keycloak              |
 | 8083      | Nginx proxy           |
 | 9000/9001 | MinIO                 |
@@ -102,4 +106,4 @@ Production deployments differ significantly from development environments. The t
 
 **Disk** – SSD; add 10 GB extra for the Docker layer cache.
 
-**MinIO** – for demos, mount a local folder instead of running MinIO to save 2 GB+ of RAM.
+**MinIO** – for demos, mount a local folder instead of running MinIO to save 2 GB+ of RAM.

env.d/development/common.dist

@@ -53,3 +53,5 @@ OIDC_AUTH_REQUEST_EXTRA_PARAMS={"acr_values": "eidas1"}
 # AI_BASE_URL=https://openaiendpoint.com
 AI_API_KEY=password
 # AI_MODEL=llama
+
+ML_FLOW_TRACKING_URI = "http://ml-flow:5050"

env.d/development/common.e2e.dist

@@ -2,7 +2,3 @@
 BURST_THROTTLE_RATES="200/minute"
 DJANGO_SERVER_TO_SERVER_API_TOKENS=test-e2e
 SUSTAINED_THROTTLE_RATES="200/hour"
-
-# Features
-FEATURE_FLAG_WEB_SEARCH=ENABLED
-FEATURE_FLAG_DOCUMENT_UPLOAD=ENABLED

renovate.json

@@ -15,24 +15,6 @@
       "matchPackageNames": ["redis"],
       "allowedVersions": "<6.0.0"
     },
-    {
-      "groupName": "ignore recent markitdown versions",
-      "matchManagers": ["pep621"],
-      "matchPackageNames": ["markitdown"],
-      "allowedVersions": "==0.0.2"
-    },
-    {
-      "groupName": "ignore recent lxml versions not supported by htmldate==1.9.3",
-      "matchManagers": ["pep621"],
-      "matchPackageNames": ["lxml"],
-      "allowedVersions": "<6"
-    },
-    {
-      "groupName": "ignore recent pylint versions not supported by pylint-django",
-      "matchManagers": ["pep621"],
-      "matchPackageNames": ["pylint"],
-      "allowedVersions": "<4"
-    },
     {
       "enabled": false,
       "groupName": "ignored js dependencies",
@@ -44,12 +26,6 @@
         "node-fetch",
         "workbox-webpack-plugin"
       ]
-    },
-    {
-      "groupName": "ignore Vercel SDK >= 5.0.0",
-      "matchManagers": ["npm"],
-      "matchPackageNames": ["@ai-sdk/react", "@ai-sdk/ui-utils"],
-      "allowedVersions": "^1.2.0"
     }
   ]
 }

src/backend/.pylintrc

@@ -23,9 +23,7 @@ jobs=0
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
-load-plugins=pylint_django,
-             pylint.extensions.no_self_use,
-             pylint_pydantic,
+load-plugins=pylint_django,pylint.extensions.no_self_use
 
 # Pickle collected data for later comparisons.
 persistent=yes

src/backend/activation_codes/admin.py

@@ -1,349 +0,0 @@
-"""Admin classes for activation codes application."""
-
-from django.conf import settings
-from django.contrib import admin
-from django.utils.html import format_html, format_html_join
-from django.utils.translation import gettext_lazy as _
-
-from . import models
-
-
-@admin.register(models.ActivationCode)
-class ActivationCodeAdmin(admin.ModelAdmin):
-    """Admin class for ActivationCode model"""
-
-    list_display = (
-        "code",
-        "usage_display",
-        "is_active",
-        "expires_at",
-        "created_at",
-        "description_short",
-    )
-
-    list_filter = (
-        "is_active",
-        "created_at",
-        "expires_at",
-    )
-
-    search_fields = (
-        "code",
-        "description",
-    )
-
-    readonly_fields = (
-        "id",
-        "current_uses",
-        "created_at",
-        "updated_at",
-        "usage_details",
-    )
-
-    fieldsets = (
-        (
-            None,
-            {
-                "fields": (
-                    "id",
-                    "code",
-                    "description",
-                )
-            },
-        ),
-        (
-            _("Configuration"),
-            {
-                "fields": (
-                    "max_uses",
-                    "current_uses",
-                    "is_active",
-                    "expires_at",
-                )
-            },
-        ),
-        (
-            _("Usage details"),
-            {"fields": ("usage_details",)},
-        ),
-        (
-            _("Timestamps"),
-            {
-                "fields": (
-                    "created_at",
-                    "updated_at",
-                )
-            },
-        ),
-    )
-
-    actions = ["recompute_current_uses"]
-
-    def get_readonly_fields(self, request, obj=None):
-        """Make `code` readonly when editing an existing ActivationCode.
-
-        When obj is None (creation form), `code` remains editable. When obj is
-        provided (editing), add `code` to readonly fields so it cannot be
-        changed after creation.
-        """
-        # Start from the configured readonly_fields to preserve other read-only fields
-        ro_fields = list(self.readonly_fields)
-        if obj is not None:
-            ro_fields.append("code")
-        return tuple(ro_fields)
-
-    def usage_display(self, obj):
-        """Display usage statistics."""
-        max_uses = obj.max_uses if obj.max_uses > 0 else "∞"
-        if obj.current_uses >= obj.max_uses and obj.max_uses > 0:
-            color = "red"
-        elif obj.current_uses > 0:
-            color = "orange"
-        else:
-            color = "green"
-
-        return format_html(
-            '<span style="color: {};">{} / {}</span>', color, obj.current_uses, max_uses
-        )
-
-    usage_display.short_description = _("Usage")
-
-    def description_short(self, obj):
-        """Display truncated description."""
-        if obj.description:
-            return obj.description[:50] + "..." if len(obj.description) > 50 else obj.description
-        return "-"
-
-    description_short.short_description = _("Description")
-
-    def usage_details(self, obj):
-        """Display detailed usage information."""
-        usages = obj.usages.select_related("user").all()
-
-        if not usages:
-            return _("No users have used this code yet")
-
-        table_head = format_html(
-            (
-                "<table style='width: 100%; border-collapse: collapse;'>"
-                "<tr style='background-color: #f0f0f0;'>"
-                "<th style='padding: 8px; text-align: left;'>{name}</th>"
-                "<th style='padding: 8px; text-align: left;'>{title}</th>"
-                "<th style='padding: 8px; text-align: left;'>{date}</th>"
-                "</tr>"
-            ),
-            name=_("Name"),
-            title=_("Email"),
-            date=_("Date"),
-        )
-
-        rows = format_html_join(
-            "",
-            (
-                "<tr style='border-bottom: 1px solid #ddd;'>"
-                "<td style='padding: 8px;'>{name}</td>"
-                "<td style='padding: 8px;'>{email}</td>"
-                "<td style='padding: 8px;'>{created_at}</td>"
-                "</tr>"
-            ),
-            (
-                {
-                    "name": usage.user.full_name or "-",
-                    "email": usage.user.email or "-",
-                    "created_at": usage.created_at.strftime("%Y-%m-%d %H:%M"),
-                }
-                for usage in usages
-            ),
-        )
-
-        return format_html("{table_head}{rows}</table>", table_head=table_head, rows=rows)
-
-    usage_details.short_description = _("Users who used this code")
-
-    @admin.action(description=_("Recompute current uses from related activations"))
-    def recompute_current_uses(self, request, queryset):
-        """Recompute the current_uses field by counting related UserActivation objects."""
-        updated_count = 0
-        for activation_code in queryset:
-            actual_uses = activation_code.usages.count()
-            if activation_code.current_uses != actual_uses:
-                activation_code.current_uses = actual_uses
-                activation_code.save(update_fields=["current_uses", "updated_at"])
-                updated_count += 1
-
-        if updated_count == 0:
-            self.message_user(
-                request,
-                _("All selected activation codes already have correct usage counts."),
-            )
-        else:
-            self.message_user(
-                request,
-                _("Successfully recomputed usage counts for %(count)d activation code(s).")
-                % {"count": updated_count},
-            )
-
-
-@admin.register(models.UserActivation)
-class UserActivationAdmin(admin.ModelAdmin):
-    """Admin class for UserActivation model"""
-
-    list_display = (
-        "user_display",
-        "user_email",
-        "activation_code",
-        "created_at",
-    )
-
-    list_filter = ("created_at",)
-
-    search_fields = (
-        "user__email",
-        "user__full_name",
-        "activation_code__code",
-    )
-
-    readonly_fields = (
-        "id",
-        "user",
-        "activation_code",
-        "created_at",
-        "updated_at",
-    )
-
-    fieldsets = (
-        (
-            None,
-            {
-                "fields": (
-                    "id",
-                    "user",
-                    "activation_code",
-                )
-            },
-        ),
-        (
-            _("Timestamps"),
-            {
-                "fields": (
-                    "created_at",
-                    "updated_at",
-                )
-            },
-        ),
-    )
-
-    def user_display(self, obj):
-        """Display user's full name."""
-        return obj.user.full_name or str(obj.user.id)
-
-    user_display.short_description = _("User")
-
-    def user_email(self, obj):
-        """Display user's email."""
-        return obj.user.email or "-"
-
-    user_email.short_description = _("Email")
-
-    def has_add_permission(self, request):
-        """Disable manual creation of user activations."""
-        return False
-
-
-@admin.register(models.UserRegistrationRequest)
-class UserRegistrationRequestAdmin(admin.ModelAdmin):
-    """Admin class for UserRegistrationRequest model"""
-
-    list_display = (
-        "user_display",
-        "created_at",
-        "has_user_activation",
-    )
-
-    readonly_fields = (
-        "id",
-        "user",
-        "created_at",
-        "updated_at",
-        "user_activation",
-    )
-
-    search_fields = (
-        "user__email",
-        "user__full_name",
-    )
-
-    list_filter = ("created_at",)
-
-    actions = ["add_to_brevo_waiting_list", "remove_from_brevo_waiting_list"]
-
-    def user_display(self, obj):
-        """Display user's full name."""
-        return obj.user.email or str(obj.user.pk)
-
-    user_display.short_description = _("User")
-
-    def has_user_activation(self, obj):
-        """Indicate if the user has used an activation code."""
-        return obj.user_activation_id is not None
-
-    has_user_activation.boolean = True
-    has_user_activation.short_description = _("Has used activation code")
-
-    @admin.action(description=_("Add selected users to Brevo waiting list"))
-    def add_to_brevo_waiting_list(self, request, queryset):
-        """Add selected users to Brevo waiting list."""
-        # pylint: disable=import-outside-toplevel
-        from core.brevo import add_user_to_brevo_list  # noqa: PLC0415
-
-        registration_to_send = queryset.filter(
-            user_activation__isnull=True,
-        )
-
-        _total_emails = 0
-        for i in range(0, registration_to_send.count(), 150):
-            batch = registration_to_send[i : i + 150]
-            emails = [reg.user.email for reg in batch if reg.user.email]
-            if emails:
-                add_user_to_brevo_list(emails, settings.BREVO_WAITING_LIST_ID)
-                _total_emails += len(emails)
-
-        if _total_emails:
-            self.message_user(
-                request,
-                _("Added %(count)d user(s) to Brevo waiting list.") % {"count": _total_emails},
-            )
-        else:
-            self.message_user(
-                request,
-                _("No valid email address found in selected registrations."),
-                level="warning",
-            )
-
-    @admin.action(description=_("Remove selected users from Brevo waiting list"))
-    def remove_from_brevo_waiting_list(self, request, queryset):
-        """Remove selected users from Brevo waiting list."""
-        # pylint: disable=import-outside-toplevel
-        from core.brevo import remove_user_from_brevo_list  # noqa: PLC0415
-
-        registration_to_send = queryset.filter(
-            user_activation__isnull=False,
-        )
-        _total_emails = 0
-        for i in range(0, registration_to_send.count(), 150):
-            batch = registration_to_send[i : i + 150]
-            emails = [reg.user.email for reg in batch if reg.user.email]
-            if emails:
-                remove_user_from_brevo_list(emails, settings.BREVO_WAITING_LIST_ID)
-                _total_emails += len(emails)
-        if _total_emails:
-            self.message_user(
-                request,
-                _("Removed %(count)d user(s) from Brevo waiting list.") % {"count": _total_emails},
-            )
-        else:
-            self.message_user(
-                request,
-                _("No valid email address found in selected registrations."),
-                level="warning",
-            )

src/backend/activation_codes/exceptions.py

@@ -1,9 +0,0 @@
-"""Exceptions for activation code handling."""
-
-
-class InvalidCodeError(ValueError):
-    """Raised when an activation code is invalid or cannot be used."""
-
-
-class UserAlreadyActivatedError(ValueError):
-    """Raised when a user tries to activate but is already activated."""

src/backend/activation_codes/factories.py

@@ -1,29 +0,0 @@
-"""Factories for creating activation code and user activation instances for testing."""
-
-from django.utils import timezone
-
-import factory.django
-
-from core.factories import UserFactory
-
-from . import models
-
-
-class ActivationCodeFactory(factory.django.DjangoModelFactory):
-    """A factory to create activation codes for testing purposes."""
-
-    class Meta:
-        model = models.ActivationCode
-
-    code = factory.LazyAttribute(lambda x: models.generate_activation_code())
-    created_at = factory.LazyAttribute(lambda obj: timezone.now())
-
-
-class UserActivationFactory(factory.django.DjangoModelFactory):
-    """A factory to create user activations for testing purposes."""
-
-    class Meta:
-        model = models.UserActivation
-
-    user = factory.SubFactory(UserFactory)
-    activation_code = factory.SubFactory(ActivationCodeFactory)

src/backend/activation_codes/migrations/0001_initial.py

@@ -1,234 +0,0 @@
-# Generated by Django 5.2.7 on 2025-10-09 08:30
-
-import uuid
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-import activation_codes.models
-
-
-class Migration(migrations.Migration):
-    initial = True
-
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="ActivationCode",
-            fields=[
-                (
-                    "id",
-                    models.UUIDField(
-                        default=uuid.uuid4,
-                        editable=False,
-                        help_text="primary key for the record as UUID",
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="id",
-                    ),
-                ),
-                (
-                    "created_at",
-                    models.DateTimeField(
-                        auto_now_add=True,
-                        help_text="date and time at which a record was created",
-                        verbose_name="created on",
-                    ),
-                ),
-                (
-                    "updated_at",
-                    models.DateTimeField(
-                        auto_now=True,
-                        help_text="date and time at which a record was last updated",
-                        verbose_name="updated on",
-                    ),
-                ),
-                (
-                    "code",
-                    models.CharField(
-                        default=activation_codes.models.generate_activation_code,
-                        help_text="The activation code that users will enter",
-                        max_length=50,
-                        unique=True,
-                        validators=[
-                            django.core.validators.RegexValidator(
-                                message="Code must be alphanumeric and contain no spaces or special characters",
-                                regex="^[A-Z0-9]+$",
-                            )
-                        ],
-                        verbose_name="activation code",
-                    ),
-                ),
-                (
-                    "max_uses",
-                    models.PositiveIntegerField(
-                        default=1,
-                        help_text="Maximum number of times this code can be used. 0 means unlimited.",
-                        verbose_name="maximum uses",
-                    ),
-                ),
-                (
-                    "current_uses",
-                    models.PositiveIntegerField(
-                        default=0,
-                        editable=False,
-                        help_text="Number of times this code has been used",
-                        verbose_name="current uses",
-                    ),
-                ),
-                (
-                    "is_active",
-                    models.BooleanField(
-                        default=True,
-                        help_text="Whether this code can still be used",
-                        verbose_name="active",
-                    ),
-                ),
-                (
-                    "expires_at",
-                    models.DateTimeField(
-                        blank=True,
-                        help_text="Date and time when this code expires",
-                        null=True,
-                        verbose_name="expires at",
-                    ),
-                ),
-                (
-                    "description",
-                    models.TextField(
-                        blank=True,
-                        help_text="Internal description or notes about this code",
-                        verbose_name="description",
-                    ),
-                ),
-            ],
-            options={
-                "verbose_name": "activation code",
-                "verbose_name_plural": "activation codes",
-                "db_table": "activation_code",
-                "ordering": ["-created_at"],
-            },
-        ),
-        migrations.CreateModel(
-            name="UserActivation",
-            fields=[
-                (
-                    "id",
-                    models.UUIDField(
-                        default=uuid.uuid4,
-                        editable=False,
-                        help_text="primary key for the record as UUID",
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="id",
-                    ),
-                ),
-                (
-                    "created_at",
-                    models.DateTimeField(
-                        auto_now_add=True,
-                        help_text="date and time at which a record was created",
-                        verbose_name="created on",
-                    ),
-                ),
-                (
-                    "updated_at",
-                    models.DateTimeField(
-                        auto_now=True,
-                        help_text="date and time at which a record was last updated",
-                        verbose_name="updated on",
-                    ),
-                ),
-                (
-                    "activation_code",
-                    models.ForeignKey(
-                        help_text="The activation code that was used",
-                        on_delete=django.db.models.deletion.PROTECT,
-                        related_name="usages",
-                        to="activation_codes.activationcode",
-                        verbose_name="activation code",
-                    ),
-                ),
-                (
-                    "user",
-                    models.OneToOneField(
-                        help_text="The user who used the activation code",
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="activation",
-                        to=settings.AUTH_USER_MODEL,
-                        verbose_name="user",
-                    ),
-                ),
-            ],
-            options={
-                "verbose_name": "user activation",
-                "verbose_name_plural": "user activations",
-                "db_table": "user_activation",
-                "ordering": ["-created_at"],
-            },
-        ),
-        migrations.CreateModel(
-            name="UserRegistrationRequest",
-            fields=[
-                (
-                    "id",
-                    models.UUIDField(
-                        default=uuid.uuid4,
-                        editable=False,
-                        help_text="primary key for the record as UUID",
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="id",
-                    ),
-                ),
-                (
-                    "created_at",
-                    models.DateTimeField(
-                        auto_now_add=True,
-                        help_text="date and time at which a record was created",
-                        verbose_name="created on",
-                    ),
-                ),
-                (
-                    "updated_at",
-                    models.DateTimeField(
-                        auto_now=True,
-                        help_text="date and time at which a record was last updated",
-                        verbose_name="updated on",
-                    ),
-                ),
-                (
-                    "user",
-                    models.OneToOneField(
-                        help_text="The user who made the registration request",
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="registration_request",
-                        to=settings.AUTH_USER_MODEL,
-                        verbose_name="user",
-                    ),
-                ),
-                (
-                    "user_activation",
-                    models.OneToOneField(
-                        blank=True,
-                        help_text="Store if the user received an activation code and used it",
-                        null=True,
-                        on_delete=django.db.models.deletion.SET_NULL,
-                        related_name="registration_request",
-                        to="activation_codes.useractivation",
-                        verbose_name="user activation",
-                    ),
-                ),
-            ],
-            options={
-                "verbose_name": "user registration request",
-                "verbose_name_plural": "user registration requests",
-                "db_table": "user_registration_request",
-                "ordering": ["-created_at"],
-            },
-        ),
-    ]

src/backend/activation_codes/models.py

@@ -1,226 +0,0 @@
-"""
-Models for the activation codes application
-"""
-
-import logging
-import secrets
-import string
-
-from django.conf import settings
-from django.core.exceptions import ValidationError
-from django.core.validators import RegexValidator
-from django.db import IntegrityError, models, transaction
-from django.utils import timezone
-from django.utils.translation import gettext_lazy as _
-
-from core.brevo import add_user_to_brevo_list, remove_user_from_brevo_list
-from core.models import BaseModel, User
-
-from activation_codes.exceptions import InvalidCodeError, UserAlreadyActivatedError
-
-logger = logging.getLogger(__name__)
-
-
-def generate_activation_code():
-    """Generate a random 16-character activation code."""
-    alphabet = string.ascii_uppercase + string.digits
-    # Remove ambiguous characters
-    alphabet = alphabet.replace("O", "").replace("0", "").replace("I", "").replace("1", "")
-    return "".join(secrets.choice(alphabet) for _ in range(16))
-
-
-class ActivationCode(BaseModel):
-    """
-    Represents an activation code that can be used to activate user accounts.
-    """
-
-    code = models.CharField(
-        verbose_name=_("activation code"),
-        help_text=_("The activation code that users will enter"),
-        max_length=50,
-        unique=True,
-        default=generate_activation_code,
-        validators=[
-            RegexValidator(
-                regex=r"^[A-Z0-9]+$",
-                message=_("Code must be alphanumeric and contain no spaces or special characters"),
-            )
-        ],
-    )
-
-    max_uses = models.PositiveIntegerField(
-        verbose_name=_("maximum uses"),
-        help_text=_("Maximum number of times this code can be used. 0 means unlimited."),
-        default=1,
-    )
-
-    current_uses = models.PositiveIntegerField(
-        verbose_name=_("current uses"),
-        help_text=_("Number of times this code has been used"),
-        default=0,
-        editable=False,
-    )
-
-    is_active = models.BooleanField(
-        verbose_name=_("active"),
-        help_text=_("Whether this code can still be used"),
-        default=True,
-    )
-
-    expires_at = models.DateTimeField(
-        verbose_name=_("expires at"),
-        help_text=_("Date and time when this code expires"),
-        null=True,
-        blank=True,
-    )
-
-    description = models.TextField(
-        verbose_name=_("description"),
-        help_text=_("Internal description or notes about this code"),
-        blank=True,
-    )
-
-    class Meta:
-        db_table = "activation_code"
-        verbose_name = _("activation code")
-        verbose_name_plural = _("activation codes")
-        ordering = ["-created_at"]
-
-    def __str__(self):
-        """Return string representation of the activation code."""
-        return f"{self.code} ({self.current_uses}/{self.max_uses if self.max_uses > 0 else '∞'})"
-
-    def is_valid(self):
-        """Check if the code is still valid and can be used."""
-        if not self.is_active:
-            return False
-
-        if self.expires_at and self.expires_at < timezone.now():
-            return False
-
-        if self.max_uses > 0 and self.current_uses >= self.max_uses:
-            return False
-
-        return True
-
-    def can_be_used(self):
-        """Alias for is_valid() for better readability."""
-        return self.is_valid()
-
-    def use(self, user):
-        """
-        Mark this code as used by a user.
-
-        Args:
-            user: The User instance using this code
-
-        Returns:
-            UserActivation instance
-
-        Raises:
-            ValidationError: If the code cannot be used
-        """
-        with transaction.atomic():
-            # Lock the activation code row to prevent concurrent overuse.
-            locked_code = ActivationCode.objects.select_for_update().get(pk=self.pk)
-
-            if not locked_code.is_valid():
-                raise InvalidCodeError(_("This activation code is no longer valid"))
-
-            # Create activation record; rely on DB uniqueness for concurrent duplicate attempts.
-            try:
-                activation = UserActivation.objects.create(user=user, activation_code=locked_code)
-            except (IntegrityError, ValidationError) as exc:
-                # User already has an activation in a concurrent or prior transaction.
-                raise UserAlreadyActivatedError(
-                    _("You have already activated your account")
-                ) from exc
-
-            existing_registration = bool(
-                UserRegistrationRequest.objects.filter(user=user).update(user_activation=activation)
-            )
-            if existing_registration:
-                transaction.on_commit(
-                    lambda: remove_user_from_brevo_list(
-                        [user.email], settings.BREVO_WAITING_LIST_ID
-                    )
-                )
-
-            # Increment usage counter safely under the same lock.
-            locked_code.current_uses += 1
-            locked_code.save(update_fields=["current_uses", "updated_at"])
-
-            transaction.on_commit(
-                lambda: add_user_to_brevo_list([user.email], settings.BREVO_FOLLOWUP_LIST_ID)
-            )
-
-            if locked_code.max_uses > 0 and locked_code.current_uses >= locked_code.max_uses:
-                logger.warning("Activation code %s has reached its maximum uses", locked_code.code)
-
-            return activation
-
-
-class UserActivation(BaseModel):
-    """
-    Records with user used which activation code and when.
-    """
-
-    user = models.OneToOneField(
-        User,
-        verbose_name=_("user"),
-        help_text=_("The user who used the activation code"),
-        on_delete=models.CASCADE,
-        related_name="activation",
-    )
-
-    activation_code = models.ForeignKey(
-        ActivationCode,
-        verbose_name=_("activation code"),
-        help_text=_("The activation code that was used"),
-        on_delete=models.PROTECT,
-        related_name="usages",
-    )
-
-    class Meta:
-        db_table = "user_activation"
-        verbose_name = _("user activation")
-        verbose_name_plural = _("user activations")
-        ordering = ["-created_at"]
-
-    def __str__(self):
-        """Return string representation of the user activation."""
-        return f"{self.user} - {self.activation_code.code}"
-
-
-class UserRegistrationRequest(BaseModel):
-    """
-    Records of user registration requests.
-    """
-
-    user = models.OneToOneField(
-        User,
-        verbose_name=_("user"),
-        help_text=_("The user who made the registration request"),
-        on_delete=models.CASCADE,
-        related_name="registration_request",
-    )
-
-    user_activation = models.OneToOneField(
-        UserActivation,
-        verbose_name=_("user activation"),
-        help_text=_("Store if the user received an activation code and used it"),
-        on_delete=models.SET_NULL,
-        related_name="registration_request",
-        null=True,
-        blank=True,
-    )
-
-    class Meta:
-        db_table = "user_registration_request"
-        verbose_name = _("user registration request")
-        verbose_name_plural = _("user registration requests")
-        ordering = ["-created_at"]
-
-    def __str__(self):
-        """Return string representation of the user registration request."""
-        return f"Registration request by {self.user}"

src/backend/activation_codes/permissions.py

@@ -1,39 +0,0 @@
-"""Permission classes for activation codes."""
-
-from django.conf import settings
-
-from rest_framework import permissions
-
-from . import models
-
-
-class IsActivatedUser(permissions.BasePermission):
-    """
-    Permission class that checks if user has activated their account.
-
-    This permission is only enforced if ACTIVATION_REQUIRED is True in settings.
-    Staff users and users without authentication requirement are always allowed.
-    """
-
-    message = "activation-required"  # Custom message to indicate activation is required to frontend
-
-    def has_permission(self, request, view):
-        """Check if user has activated their account."""
-        # If activation is not required, allow access
-        if not settings.ACTIVATION_REQUIRED:
-            return True
-
-        # Staff users can always access
-        if request.user and request.user.is_staff:
-            return True
-
-        # Anonymous users are handled by other permission classes
-        if not request.user or not request.user.is_authenticated:
-            return True
-
-        # Check if user has an activation record
-        return models.UserActivation.objects.filter(user=request.user).exists()
-
-    def has_object_permission(self, request, view, obj):
-        """Check object-level permission."""
-        return self.has_permission(request, view)

src/backend/activation_codes/serializers.py

@@ -1,50 +0,0 @@
-"""Serializers for the activation codes application."""
-
-from django.utils.translation import gettext_lazy as _
-
-from rest_framework import serializers
-
-from . import models
-
-
-class ActivationCodeValidationSerializer(serializers.Serializer):  # pylint: disable=abstract-method
-    """Serializer for validating an activation code."""
-
-    code = serializers.CharField(
-        max_length=50, required=True, help_text=_("The activation code to validate")
-    )
-
-    def validate_code(self, value):
-        """Validate that the code exists and is valid."""
-        # Normalize the code (remove spaces, convert to uppercase)
-        return value.strip().upper().replace(" ", "").replace("-", "")
-
-
-class UserActivationSerializer(serializers.ModelSerializer):
-    """Serializer for user activation records."""
-
-    code = serializers.CharField(source="activation_code.code", read_only=True)
-    activated_at = serializers.DateTimeField(source="created_at", read_only=True)
-
-    class Meta:
-        model = models.UserActivation
-        fields = ["id", "code", "activated_at"]
-        read_only_fields = ["id", "code", "activated_at"]
-
-
-class ActivationStatusSerializer(serializers.Serializer):  # pylint: disable=abstract-method
-    """Serializer for activation status response."""
-
-    is_activated = serializers.BooleanField(read_only=True)
-    activation = UserActivationSerializer(read_only=True, allow_null=True)
-    requires_activation = serializers.BooleanField(read_only=True)
-
-
-class UserRegistrationRequestSerializer(serializers.ModelSerializer):
-    """Serializer for registering a user for activation notifications."""
-
-    user = serializers.HiddenField(default=serializers.CurrentUserDefault())
-
-    class Meta:
-        model = models.UserRegistrationRequest
-        fields = ["user"]

src/backend/activation_codes/tests/test_integration.py

@@ -1,236 +0,0 @@
-"""Integration tests for activation_codes application."""
-
-from datetime import timedelta
-
-from django.utils import timezone
-
-import pytest
-from rest_framework import status
-
-from core.factories import UserFactory
-
-from activation_codes.factories import ActivationCodeFactory
-from activation_codes.models import ActivationCode, UserActivation
-
-
-@pytest.mark.django_db
-def test_complete_activation_flow(api_client, settings):
-    """Test complete user activation flow from registration to usage."""
-    settings.ACTIVATION_REQUIRED = True
-
-    # Create a user (simulating registration)
-    user = UserFactory(email="newuser@example.com", password="password123")
-
-    # Create an activation code (simulating admin creating codes)
-    activation_code = ActivationCode.objects.create(
-        code="WELCOME123456789", max_uses=10, description="Welcome batch for new users"
-    )
-
-    # User logs in
-    api_client.force_authenticate(user=user)
-
-    # Step 1: Check activation status (should not be activated)
-    response = api_client.get("/api/v1.0/activation/status/")
-    assert response.status_code == status.HTTP_200_OK
-    assert response.data["is_activated"] is False
-    assert response.data["requires_activation"] is True
-
-    # Step 2: User enters activation code
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "WELCOME123456789"})
-    assert response.status_code == status.HTTP_201_CREATED
-    assert "successfully activated" in response.data["detail"]
-
-    # Step 3: Check activation status again (should now be activated)
-    response = api_client.get("/api/v1.0/activation/status/")
-    assert response.status_code == status.HTTP_200_OK
-    assert response.data["is_activated"] is True
-    assert response.data["activation"]["code"] == "WELCOME123456789"
-
-    # Step 4: Verify in database
-    assert UserActivation.objects.filter(user=user).exists()
-    activation_code.refresh_from_db()
-    assert activation_code.current_uses == 1
-
-
-@pytest.mark.django_db
-def test_activation_not_required_flow(api_client, settings):
-    """Test that when activation is not required, users can access without codes."""
-    settings.ACTIVATION_REQUIRED = False
-
-    user = UserFactory(email="freeuser@example.com", password="password123")
-
-    api_client.force_authenticate(user=user)
-
-    # Check status
-    response = api_client.get("/api/v1.0/activation/status/")
-    assert response.status_code == status.HTTP_200_OK
-    assert response.data["requires_activation"] is False
-    assert response.data["is_activated"] is False  # Not activated but not required
-
-
-@pytest.mark.django_db
-def test_multiple_users_same_code(api_client, settings):
-    """Test multiple users using the same multi-use code."""
-    settings.ACTIVATION_REQUIRED = True
-
-    # Create a multi-use code
-    code = ActivationCode.objects.create(
-        code="TEAMCODE12345678", max_uses=3, description="Team activation code"
-    )
-
-    # Create 3 users
-    users = []
-    for i in range(3):
-        user = UserFactory(email=f"teamuser{i}@example.com", password="password123")
-        users.append(user)
-
-    # Each user activates
-    for i, user in enumerate(users):
-        api_client.force_authenticate(user=user)
-        response = api_client.post("/api/v1.0/activation/validate/", {"code": "TEAMCODE12345678"})
-        assert response.status_code == status.HTTP_201_CREATED
-
-        code.refresh_from_db()
-        assert code.current_uses == i + 1
-
-    # Code should now be exhausted
-    code.refresh_from_db()
-    assert code.is_valid() is False
-
-    # Try with a 4th user (should fail)
-    user4 = UserFactory(email="teamuser4@example.com", password="password123")
-    api_client.force_authenticate(user=user4)
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "TEAMCODE12345678"})
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-
-@pytest.mark.django_db
-def test_code_expiration_scenario(api_client, settings):
-    """Test code expiration over time."""
-    settings.ACTIVATION_REQUIRED = True
-
-    # Create a code that expires in 1 day
-    future_time = timezone.now() + timedelta(days=1)
-    _code = ActivationCode.objects.create(code="EXPIRES123456789", expires_at=future_time)
-
-    user = UserFactory(email="timeduser@example.com", password="password123")
-    api_client.force_authenticate(user=user)
-
-    # Should work now
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "EXPIRES123456789"})
-    assert response.status_code == status.HTTP_201_CREATED
-
-
-@pytest.mark.django_db
-def test_staff_user_bypass(api_client, settings):
-    """Test that staff users bypass activation requirement."""
-    settings.ACTIVATION_REQUIRED = True
-
-    staff_user = UserFactory(email="staff@example.com", password="password123", is_staff=True)
-
-    api_client.force_authenticate(user=staff_user)
-
-    # Staff should be able to check status even without activation
-    response = api_client.get("/api/v1.0/activation/status/")
-    assert response.status_code == status.HTTP_200_OK
-
-
-@pytest.mark.django_db
-def test_user_cannot_activate_twice(api_client, settings):
-    """Test that a user cannot activate their account twice."""
-    settings.ACTIVATION_REQUIRED = True
-
-    user = UserFactory(email="onceuser@example.com", password="password123")
-
-    _code1 = ActivationCodeFactory(code="FIRST12345678901")
-    _code2 = ActivationCodeFactory(code="SECOND1234567890")
-
-    api_client.force_authenticate(user=user)
-
-    # First activation
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "FIRST12345678901"})
-    assert response.status_code == status.HTTP_201_CREATED
-
-    # Try second activation
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "SECOND1234567890"})
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-    assert response.data == {"code": "account-already-activated"}
-
-    # Verify only one activation exists
-    assert UserActivation.objects.filter(user=user).count() == 1
-
-
-@pytest.mark.django_db
-def test_code_variations_normalized(api_client, settings):
-    """Test that different code input formats are normalized correctly."""
-    settings.ACTIVATION_REQUIRED = True
-
-    code = ActivationCodeFactory(code="TESTCODE12345678")
-
-    test_cases = [
-        "testcode12345678",  # lowercase
-        "TESTCODE12345678",  # uppercase
-        "test code 1234 5678",  # with spaces
-        "TEST-CODE-1234-5678",  # with dashes
-        " test-code 1234-5678 ",  # mixed with leading/trailing spaces
-    ]
-
-    for i, code_variation in enumerate(test_cases):
-        user = UserFactory(email=f"varuser{i}@example.com", password="password123")
-
-        # Update code to allow multiple uses
-        code.max_uses = 0
-        code.save()
-
-        api_client.force_authenticate(user=user)
-        response = api_client.post("/api/v1.0/activation/validate/", {"code": code_variation})
-        assert response.status_code == status.HTTP_201_CREATED, (
-            f"Failed for variation: {code_variation}"
-        )
-
-
-@pytest.mark.django_db
-def test_inactive_code_cannot_be_used(api_client, settings):
-    """Test that inactive codes cannot be used even if valid otherwise."""
-    settings.ACTIVATION_REQUIRED = True
-
-    _code = ActivationCodeFactory(
-        code="INACTIVE123VALID",
-        is_active=False,
-        max_uses=10,
-        expires_at=timezone.now() + timedelta(days=30),
-    )
-
-    user = UserFactory(email="blockeduser@example.com", password="password123")
-
-    api_client.force_authenticate(user=user)
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "INACTIVE123VALID"})
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-
-@pytest.mark.django_db
-def test_concurrent_activations_multi_use_code(api_client, settings):
-    """Test that concurrent activations don't exceed max_uses."""
-    settings.ACTIVATION_REQUIRED = True
-
-    code = ActivationCodeFactory(code="CONCURRENT123456", max_uses=2)
-
-    # Create 3 users
-    users = [
-        UserFactory(email=f"concurrent{i}@example.com", password="password123") for i in range(3)
-    ]
-
-    # First two should succeed
-    for i in range(2):
-        api_client.force_authenticate(user=users[i])
-        response = api_client.post("/api/v1.0/activation/validate/", {"code": "CONCURRENT123456"})
-        assert response.status_code == status.HTTP_201_CREATED
-
-    # Third should fail
-    api_client.force_authenticate(user=users[2])
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "CONCURRENT123456"})
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-    # Verify only 2 activations
-    code.refresh_from_db()
-    assert code.current_uses == 2

src/backend/activation_codes/tests/test_models.py

@@ -1,328 +0,0 @@
-"""Tests for activation_codes models."""
-
-import json
-from datetime import timedelta
-
-from django.core.exceptions import ValidationError
-from django.db.models import ProtectedError
-from django.utils import timezone
-
-import pytest
-import responses
-
-from core.factories import UserFactory
-
-from activation_codes.exceptions import InvalidCodeError, UserAlreadyActivatedError
-from activation_codes.factories import ActivationCodeFactory, UserActivationFactory
-from activation_codes.models import (
-    ActivationCode,
-    UserActivation,
-    UserRegistrationRequest,
-    generate_activation_code,
-)
-
-
-@pytest.mark.django_db
-def test_generate_activation_code():
-    """Test that generate_activation_code creates a valid code."""
-    code = generate_activation_code()
-
-    assert len(code) == 16
-    assert code.isupper()
-    assert all(c.isalnum() for c in code)
-    # Check that ambiguous characters are not present
-    assert "O" not in code
-    assert "0" not in code
-    assert "I" not in code
-    assert "1" not in code
-
-
-@pytest.mark.django_db
-def test_generate_activation_code_uniqueness():
-    """Test that generated codes are unique."""
-    codes = [generate_activation_code() for _ in range(100)]
-    assert len(codes) == len(set(codes))
-
-
-@pytest.mark.django_db
-def test_activation_code_creation():
-    """Test creating an activation code."""
-    activation_code = ActivationCodeFactory(code="TEST1234ABCD5678")
-
-    assert activation_code.code == "TEST1234ABCD5678"
-    assert activation_code.max_uses == 1
-    assert activation_code.current_uses == 0
-    assert activation_code.is_active is True
-    assert activation_code.expires_at is None
-
-
-@pytest.mark.django_db
-def test_activation_code_auto_generated_code():
-    """Test that activation code is auto-generated if not provided."""
-    code = ActivationCodeFactory()
-    assert len(code.code) == 16
-    assert code.code.isupper()
-
-
-@pytest.mark.django_db
-def test_activation_code_str_representation():
-    """Test string representation of activation code."""
-    activation_code = ActivationCodeFactory(code="TEST1234ABCD5678")
-    assert str(activation_code) == "TEST1234ABCD5678 (0/1)"
-
-
-@pytest.mark.django_db
-def test_activation_code_str_representation_unlimited():
-    """Test string representation of unlimited activation code."""
-    unlimited_activation_code = ActivationCodeFactory(code="UNLIMITED123CODE", max_uses=0)
-
-    assert str(unlimited_activation_code) == "UNLIMITED123CODE (0/∞)"
-
-
-@pytest.mark.django_db
-def test_activation_code_is_valid_active():
-    """Test that an active, non-expired code is valid."""
-    activation_code = ActivationCodeFactory()
-    assert activation_code.is_valid() is True
-    assert activation_code.can_be_used() is True
-
-
-@pytest.mark.django_db
-def test_activation_code_is_valid_inactive():
-    """Test that an inactive code is not valid."""
-    inactive_activation_code = ActivationCodeFactory(is_active=False)
-    assert inactive_activation_code.is_valid() is False
-    assert inactive_activation_code.can_be_used() is False
-
-
-@pytest.mark.django_db
-def test_activation_code_is_valid_expired():
-    """Test that an expired code is not valid."""
-    expired_activation_code = ActivationCodeFactory(
-        created_at=timezone.now() - timedelta(days=10),
-        expires_at=timezone.now() - timedelta(days=1),
-    )
-    assert expired_activation_code.is_valid() is False
-    assert expired_activation_code.can_be_used() is False
-
-
-@pytest.mark.django_db
-def test_activation_code_is_valid_max_uses_reached():
-    """Test that a code with max uses reached is not valid."""
-    activation_code = ActivationCodeFactory(max_uses=1)
-    activation_code.current_uses = 1
-    activation_code.save()
-    assert activation_code.is_valid() is False
-
-
-@pytest.mark.django_db
-def test_activation_code_is_valid_unlimited_uses():
-    """Test that unlimited code is always valid regardless of current uses."""
-    unlimited_activation_code = ActivationCodeFactory(max_uses=0)
-    unlimited_activation_code.current_uses = 100
-    unlimited_activation_code.save()
-    assert unlimited_activation_code.is_valid() is True
-
-
-@pytest.mark.django_db
-def test_activation_code_use_success():
-    """Test successfully using an activation code."""
-    user = UserFactory()
-    activation_code = ActivationCodeFactory()
-    activation = activation_code.use(user)
-
-    assert isinstance(activation, UserActivation)
-    assert activation.user == user
-    assert activation.activation_code == activation_code
-
-    # Check that usage counter was incremented
-    activation_code.refresh_from_db()
-    assert activation_code.current_uses == 1
-
-
-@pytest.mark.django_db
-def test_activation_code_use_invalid_code():
-    """Test using an invalid activation code raises error."""
-    inactive_activation_code = ActivationCodeFactory(is_active=False)
-    user = UserFactory()
-    with pytest.raises(InvalidCodeError):
-        inactive_activation_code.use(user)
-
-
-@pytest.mark.django_db
-def test_activation_code_use_already_activated():
-    """Test using a code when user is already activated raises error."""
-    user = UserFactory()
-    activation_code = ActivationCodeFactory()
-
-    # First activation
-    activation_code.use(user)
-
-    # Try to activate again with a different code
-    another_code = ActivationCodeFactory(code="ANOTHER123456789")
-    with pytest.raises(UserAlreadyActivatedError):
-        another_code.use(user)
-
-
-@pytest.mark.django_db
-def test_activation_code_use_multi_use():
-    """Test using a multi-use activation code."""
-    multi_use_activation_code = ActivationCodeFactory(max_uses=4)
-    users = [UserFactory(email=f"user{i}@example.com") for i in range(3)]
-
-    for i, user in enumerate(users):
-        activation = multi_use_activation_code.use(user)
-        assert activation.user == user
-
-        multi_use_activation_code.refresh_from_db()
-        assert multi_use_activation_code.current_uses == i + 1
-
-    # Code should still be valid
-    assert multi_use_activation_code.is_valid() is True
-
-
-@pytest.mark.django_db
-def test_activation_code_use_max_uses_exceeded():
-    """Test that code cannot be used when max uses is reached."""
-    user = UserFactory()
-    activation_code = ActivationCodeFactory(max_uses=1)
-
-    # Use the code
-    activation_code.use(user)
-
-    # Try to use it again with another user
-    another_user = UserFactory(email="another@example.com")
-    with pytest.raises(InvalidCodeError):
-        activation_code.use(another_user)
-
-
-@pytest.mark.django_db
-def test_activation_code_expiration():
-    """Test that code expires correctly."""
-    future_expiry = timezone.now() + timedelta(days=1)
-    code = ActivationCodeFactory(code="FUTURE123456789", expires_at=future_expiry)
-
-    assert code.is_valid() is True
-
-    # Manually set to past
-    code.expires_at = timezone.now() - timedelta(seconds=1)
-    code.save()
-
-    assert code.is_valid() is False
-
-
-@pytest.mark.django_db
-def test_user_activation_str_representation():
-    """Test string representation of user activation."""
-    user_activation = UserActivationFactory(activation_code__code="TEST1234ABCD5678")
-
-    expected = f"{user_activation.user} - TEST1234ABCD5678"
-    assert str(user_activation) == expected
-
-
-@pytest.mark.django_db
-def test_user_activation_one_to_one_relationship():
-    """Test that a user can only have one activation."""
-    user_activation = UserActivationFactory()
-
-    # Try to create another activation for the same user
-    with pytest.raises(ValidationError):  # should be IntegrityError
-        UserActivationFactory(user=user_activation.user)
-
-
-@pytest.mark.django_db
-def test_activation_code_protect_on_delete():
-    """Test that activation code is protected from deletion when used."""
-    user_activation = UserActivationFactory()
-
-    # Try to delete the activation code
-    with pytest.raises(ProtectedError):
-        user_activation.activation_code.delete()
-
-
-@pytest.mark.django_db
-def test_user_activation_cascade_on_user_delete():
-    """Test that activation is deleted when user is deleted."""
-    activation = UserActivationFactory()
-    activation_id = activation.pk
-
-    activation.user.delete()
-
-    # Activation should be deleted
-    assert not UserActivation.objects.filter(id=activation_id).exists()
-
-
-@pytest.mark.django_db
-def test_activation_code_ordering():
-    """Test that activation codes are ordered by created_at descending."""
-    code1 = ActivationCodeFactory(code="CODE1")
-    code2 = ActivationCodeFactory(code="CODE2")
-    code3 = ActivationCodeFactory(code="CODE3")
-
-    codes = list(ActivationCode.objects.all())
-    assert codes == [code3, code2, code1]
-
-
-@pytest.mark.django_db
-def test_user_activation_ordering():
-    """Test that user activations are ordered by created_at descending."""
-    code1 = ActivationCodeFactory(code="CODE1", max_uses=3)
-    code2 = ActivationCodeFactory(code="CODE2", max_uses=3)
-
-    user1 = UserFactory(email="user1@example.com")
-    user2 = UserFactory(email="user2@example.com")
-
-    activation1 = UserActivationFactory(user=user1, activation_code=code1)
-    activation2 = UserActivationFactory(user=user2, activation_code=code2)
-
-    activations = list(UserActivation.objects.all())
-    assert activations == [activation2, activation1]
-
-
-@responses.activate
-@pytest.mark.django_db(transaction=True)
-def test_activation_code_use_success_notify_brevo(settings):
-    """Test successfully using an activation code and notify Brevo."""
-    settings.BREVO_API_KEY = "test_brevo_api_key"
-    settings.BREVO_WAITING_LIST_ID = "test_waiting_list_id"
-    settings.BREVO_FOLLOWUP_LIST_ID = "test_followup_list_name"
-
-    brevo_remove_mock = responses.post(
-        "https://api.brevo.com/v3/contacts/lists/test_waiting_list_id/contacts/remove",
-        json={"message": "Contacts added successfully"},
-        status=201,
-    )
-
-    brevo_create_contact = responses.post(
-        "https://api.brevo.com/v3/contacts",
-        status=200,
-    )
-
-    brevo_add_mock = responses.post(
-        "https://api.brevo.com/v3/contacts/lists/test_followup_list_name/contacts/add",
-        json={"message": "Contacts added successfully"},
-        status=201,
-    )
-
-    user = UserFactory()
-    registration = UserRegistrationRequest.objects.create(user=user)
-    activation_code = ActivationCodeFactory()
-    activation = activation_code.use(user)
-
-    registration.refresh_from_db()
-    assert registration.user_activation == activation
-
-    assert len(brevo_remove_mock.calls) == 1
-    assert brevo_remove_mock.calls[0].request.headers["api-key"] == "test_brevo_api_key"
-    assert json.loads(brevo_remove_mock.calls[0].request.body) == {"emails": [user.email]}
-
-    assert len(brevo_create_contact.calls) == 1
-    assert brevo_create_contact.calls[0].request.headers["api-key"] == "test_brevo_api_key"
-    assert json.loads(brevo_create_contact.calls[0].request.body) == {
-        "email": user.email,
-        "updateEnabled": True,
-    }
-
-    assert len(brevo_add_mock.calls) == 1
-    assert brevo_add_mock.calls[0].request.headers["api-key"] == "test_brevo_api_key"
-    assert json.loads(brevo_add_mock.calls[0].request.body) == {"emails": [user.email]}

src/backend/activation_codes/tests/test_permissions.py

@@ -1,133 +0,0 @@
-"""Tests for activation_codes permissions."""
-
-from django.test import RequestFactory
-
-import pytest
-from rest_framework.views import APIView
-
-from core.factories import UserFactory
-
-from activation_codes.factories import UserActivationFactory
-from activation_codes.permissions import IsActivatedUser
-
-
-@pytest.fixture(name="request_factory")
-def request_factory_fixture():
-    """Fixture to provide a request factory."""
-    return RequestFactory()
-
-
-@pytest.fixture(name="view")
-def view_fixture():
-    """Fixture to provide a basic view instance."""
-    return APIView()
-
-
-@pytest.mark.django_db
-def test_is_activated_user_permission_activation_not_required(request_factory, view, settings):
-    """Test that permission allows access when activation is not required."""
-    settings.ACTIVATION_REQUIRED = False
-    user = UserFactory()
-
-    request = request_factory.get("/")
-    request.user = user
-
-    permission = IsActivatedUser()
-    assert permission.has_permission(request, view) is True
-
-
-@pytest.mark.django_db
-def test_is_activated_user_permission_staff_user(request_factory, view, settings):
-    """Test that staff users always have permission."""
-    settings.ACTIVATION_REQUIRED = True
-    staff_user = UserFactory(email="staff@example.com", password="password123", is_staff=True)
-
-    request = request_factory.get("/")
-    request.user = staff_user
-
-    permission = IsActivatedUser()
-    assert permission.has_permission(request, view) is True
-
-
-@pytest.mark.django_db
-def test_is_activated_user_permission_anonymous_user(request_factory, view, settings):
-    """Test that anonymous users are allowed (handled by other permissions)."""
-    settings.ACTIVATION_REQUIRED = True
-
-    request = request_factory.get("/")
-    request.user = None
-
-    permission = IsActivatedUser()
-    assert permission.has_permission(request, view) is True
-
-
-@pytest.mark.django_db
-def test_is_activated_user_permission_activated_user(request_factory, view, settings):
-    """Test that activated users have permission."""
-    settings.ACTIVATION_REQUIRED = True
-
-    # Activate the user
-    activation = UserActivationFactory()
-
-    request = request_factory.get("/")
-    request.user = activation.user
-
-    permission = IsActivatedUser()
-    assert permission.has_permission(request, view) is True
-
-
-@pytest.mark.django_db
-def test_is_activated_user_permission_not_activated_user(request_factory, view, settings):
-    """Test that non-activated users do not have permission."""
-    settings.ACTIVATION_REQUIRED = True
-
-    user = UserFactory()
-
-    request = request_factory.get("/")
-    request.user = user
-
-    permission = IsActivatedUser()
-    assert permission.has_permission(request, view) is False
-
-
-@pytest.mark.django_db
-def test_is_activated_user_permission_custom_message():
-    """Test that permission has custom message for frontend."""
-    permission = IsActivatedUser()
-    assert permission.message == "activation-required"
-
-
-@pytest.mark.django_db
-def test_is_activated_user_object_permission(request_factory, view, settings):
-    """Test object-level permission delegates to has_permission."""
-    settings.ACTIVATION_REQUIRED = True
-
-    _user = UserFactory()
-
-    # Activate the user
-    activation = UserActivationFactory()
-
-    request = request_factory.get("/")
-    request.user = activation.user
-
-    permission = IsActivatedUser()
-    obj = object()  # Any object
-
-    # Object permission should delegate to has_permission
-    assert permission.has_object_permission(request, view, obj) is True
-
-
-@pytest.mark.django_db
-def test_is_activated_user_object_permission_not_activated(request_factory, view, settings):
-    """Test object-level permission when user is not activated."""
-    settings.ACTIVATION_REQUIRED = True
-
-    user = UserFactory()
-
-    request = request_factory.get("/")
-    request.user = user
-
-    permission = IsActivatedUser()
-    obj = object()
-
-    assert permission.has_object_permission(request, view, obj) is False

src/backend/activation_codes/tests/test_serializers.py

@@ -1,150 +0,0 @@
-"""Tests for activation_codes serializers."""
-
-import pytest
-
-from activation_codes.factories import ActivationCodeFactory, UserActivationFactory
-from activation_codes.serializers import (
-    ActivationCodeValidationSerializer,
-    ActivationStatusSerializer,
-    UserActivationSerializer,
-)
-
-
-@pytest.mark.django_db
-def test_activation_code_validation_serializer_valid_code():
-    """Test validating a valid activation code."""
-    # Create a valid activation code
-    _activation_code = ActivationCodeFactory(code="TEST1234ABCD5678")
-
-    serializer = ActivationCodeValidationSerializer(data={"code": "TEST1234ABCD5678"})
-    assert serializer.is_valid()
-    assert serializer.validated_data["code"] == "TEST1234ABCD5678"
-
-
-@pytest.mark.django_db
-def test_activation_code_validation_serializer_normalize_lowercase():
-    """Test that code is normalized to uppercase."""
-    # Create a valid activation code
-    _activation_code = ActivationCodeFactory(code="TEST1234ABCD5678")
-
-    serializer = ActivationCodeValidationSerializer(data={"code": "test1234abcd5678"})
-    assert serializer.is_valid()
-    assert serializer.validated_data["code"] == "TEST1234ABCD5678"
-
-
-@pytest.mark.django_db
-def test_activation_code_validation_serializer_normalize_with_spaces():
-    """Test that spaces are removed from code."""
-    # Create a valid activation code
-    _activation_code = ActivationCodeFactory(code="TEST1234ABCD5678")
-
-    serializer = ActivationCodeValidationSerializer(data={"code": "TEST 1234 ABCD 5678"})
-    assert serializer.is_valid()
-    assert serializer.validated_data["code"] == "TEST1234ABCD5678"
-
-
-@pytest.mark.django_db
-def test_activation_code_validation_serializer_normalize_with_dashes():
-    """Test that dashes are removed from code."""
-    # Create a valid activation code
-    _activation_code = ActivationCodeFactory(code="TEST1234ABCD5678")
-
-    serializer = ActivationCodeValidationSerializer(data={"code": "TEST-1234-ABCD-5678"})
-    assert serializer.is_valid()
-    assert serializer.validated_data["code"] == "TEST1234ABCD5678"
-
-
-@pytest.mark.django_db
-def test_activation_code_validation_serializer_normalize_mixed():
-    """Test that code with spaces, dashes and lowercase is normalized."""
-    # Create a valid activation code
-    _activation_code = ActivationCodeFactory(code="TEST1234ABCD5678")
-
-    serializer = ActivationCodeValidationSerializer(data={"code": " test-1234 abcd-5678 "})
-    assert serializer.is_valid()
-    assert serializer.validated_data["code"] == "TEST1234ABCD5678"
-
-
-@pytest.mark.django_db
-def test_activation_code_validation_serializer_missing_code():
-    """Test that code field is required."""
-    serializer = ActivationCodeValidationSerializer(data={})
-    assert not serializer.is_valid()
-    assert "code" in serializer.errors
-
-
-@pytest.mark.django_db
-def test_user_activation_serializer():
-    """Test serializing a user activation."""
-    activation = UserActivationFactory(activation_code__code="TEST1234ABCD5678")
-
-    serializer = UserActivationSerializer(activation)
-    data = serializer.data
-
-    assert "id" in data
-    assert data["code"] == "TEST1234ABCD5678"
-    assert "activated_at" in data
-    assert data["activated_at"] is not None
-
-
-@pytest.mark.django_db
-def test_user_activation_serializer_read_only_fields():
-    """Test that all fields are read-only."""
-    activation = UserActivationFactory()
-
-    serializer = UserActivationSerializer(activation)
-
-    # All fields should be in read_only_fields
-    meta = serializer.Meta
-    assert set(meta.read_only_fields) == set(meta.fields)
-
-
-@pytest.mark.django_db
-def test_activation_status_serializer_activated():
-    """Test serializing activation status for activated user."""
-    activation = UserActivationFactory(activation_code__code="TEST1234ABCD5678")
-
-    data = {"is_activated": True, "activation": activation, "requires_activation": True}
-
-    serializer = ActivationStatusSerializer(data)
-    serialized_data = serializer.data
-
-    assert serialized_data["is_activated"] is True
-    assert serialized_data["activation"] is not None
-    assert serialized_data["activation"]["code"] == "TEST1234ABCD5678"
-    assert serialized_data["requires_activation"] is True
-
-
-@pytest.mark.django_db
-def test_activation_status_serializer_not_activated():
-    """Test serializing activation status for non-activated user."""
-    data = {"is_activated": False, "activation": None, "requires_activation": True}
-
-    serializer = ActivationStatusSerializer(data)
-    serialized_data = serializer.data
-
-    assert serialized_data["is_activated"] is False
-    assert serialized_data["activation"] is None
-    assert serialized_data["requires_activation"] is True
-
-
-@pytest.mark.django_db
-def test_activation_status_serializer_activation_not_required():
-    """Test serializing activation status when activation is not required."""
-    data = {"is_activated": False, "activation": None, "requires_activation": False}
-
-    serializer = ActivationStatusSerializer(data)
-    serialized_data = serializer.data
-
-    assert serialized_data["is_activated"] is False
-    assert serialized_data["activation"] is None
-    assert serialized_data["requires_activation"] is False
-
-
-@pytest.mark.django_db
-def test_activation_status_serializer_all_fields_read_only():
-    """Test that all fields in ActivationStatusSerializer are read-only."""
-    serializer = ActivationStatusSerializer()
-
-    for field_name, field in serializer.fields.items():
-        assert field.read_only is True, f"Field {field_name} should be read-only"

src/backend/activation_codes/tests/test_viewsets.py

@@ -1,411 +0,0 @@
-"""Tests for activation_codes viewsets."""
-
-import json
-from datetime import timedelta
-from unittest.mock import patch
-
-from django.utils import timezone
-
-import pytest
-import responses
-from rest_framework import status
-
-from core.factories import UserFactory
-
-from activation_codes.factories import ActivationCodeFactory, UserActivationFactory
-from activation_codes.models import ActivationCode, UserActivation, UserRegistrationRequest
-
-
-@pytest.mark.django_db
-def test_activation_status_unauthenticated(api_client):
-    """Test that unauthenticated users cannot access status endpoint."""
-    response = api_client.get("/api/v1.0/activation/status/")
-    assert response.status_code == status.HTTP_401_UNAUTHORIZED
-
-
-@pytest.mark.django_db
-def test_activation_status_authenticated_not_activated(api_client, settings):
-    """Test activation status for authenticated but not activated user."""
-    settings.ACTIVATION_REQUIRED = True
-
-    user = UserFactory()
-    api_client.force_authenticate(user=user)
-
-    response = api_client.get("/api/v1.0/activation/status/")
-
-    assert response.status_code == status.HTTP_200_OK
-    assert response.data["is_activated"] is False
-    assert response.data["activation"] is None
-    assert response.data["requires_activation"] is True
-
-
-@pytest.mark.django_db
-def test_activation_status_authenticated_activated(api_client, settings):
-    """Test activation status for activated user."""
-    settings.ACTIVATION_REQUIRED = True
-    activation = UserActivationFactory(activation_code__code="TEST1234ABCD5678")
-    api_client.force_authenticate(user=activation.user)
-
-    response = api_client.get("/api/v1.0/activation/status/")
-
-    assert response.status_code == status.HTTP_200_OK
-    assert response.data["is_activated"] is True
-    assert response.data["activation"] is not None
-    assert response.data["activation"]["code"] == "TEST1234ABCD5678"
-    assert "activated_at" in response.data["activation"]
-    assert response.data["requires_activation"] is True
-
-
-@pytest.mark.django_db
-def test_activation_status_activation_not_required(api_client, settings):
-    """Test activation status when activation is not required."""
-    settings.ACTIVATION_REQUIRED = False
-    user = UserFactory()
-    api_client.force_authenticate(user=user)
-
-    response = api_client.get("/api/v1.0/activation/status/")
-
-    assert response.status_code == status.HTTP_200_OK
-    assert response.data["requires_activation"] is False
-
-
-@pytest.mark.django_db
-def test_validate_code_unauthenticated(api_client):
-    """Test that unauthenticated users cannot validate codes."""
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "TEST1234ABCD5678"})
-    assert response.status_code == status.HTTP_401_UNAUTHORIZED
-
-
-@pytest.mark.django_db
-def test_validate_code_success(api_client):
-    """Test successfully validating and using an activation code."""
-    user = UserFactory()
-    activation_code = ActivationCode.objects.create(code="TEST1234ABCD5678")
-    api_client.force_authenticate(user=user)
-
-    with patch("activation_codes.viewsets.logger") as mock_logger:
-        response = api_client.post("/api/v1.0/activation/validate/", {"code": "TEST1234ABCD5678"})
-
-    assert response.status_code == status.HTTP_201_CREATED
-    assert "Your account has been successfully activated" in response.data["detail"]
-    assert "activation" in response.data
-    assert response.data["activation"]["code"] == "TEST1234ABCD5678"
-
-    # Verify user is now activated
-    assert UserActivation.objects.filter(user=user).exists()
-
-    # Verify activation code was used
-    activation_code.refresh_from_db()
-    assert activation_code.current_uses == 1
-
-    # Verify logging
-    mock_logger.info.assert_called_once()
-
-
-@pytest.mark.django_db
-def test_validate_code_with_spaces_and_lowercase(api_client):
-    """Test validating code with spaces and lowercase."""
-    user = UserFactory()
-    ActivationCodeFactory(code="TEST1234ABCD5678")
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "test 1234 abcd 5678"})
-
-    assert response.status_code == status.HTTP_201_CREATED
-    assert UserActivation.objects.filter(user=user).exists()
-
-
-@pytest.mark.django_db
-def test_validate_code_already_activated(api_client):
-    """Test validating code when user is already activated."""
-    # First activation
-    activation = UserActivationFactory()
-    api_client.force_authenticate(user=activation.user)
-
-    # Try to activate again with different code
-    _another_code = ActivationCodeFactory(code="ANOTHER123456789")
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "ANOTHER123456789"})
-
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-    assert response.data == {"code": "account-already-activated"}
-
-
-@pytest.mark.django_db
-def test_validate_code_nonexistent(api_client):
-    """Test validating a non-existent code."""
-    user = UserFactory()
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "NONEXISTENT12345"})
-
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-    assert response.data == {"code": "invalid-code"}
-
-
-@pytest.mark.django_db
-def test_validate_code_invalid_serializer(api_client):
-    """Test validating with invalid data."""
-    user = UserFactory()
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post(
-        "/api/v1.0/activation/validate/",
-        {},  # Missing code
-    )
-
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-    assert "code" in response.data
-
-
-@pytest.mark.django_db
-def test_validate_code_inactive(api_client):
-    """Test validating an inactive code."""
-    user = UserFactory()
-    ActivationCodeFactory(code="INACTIVE12345678", is_active=False)
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "INACTIVE12345678"})
-
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-    assert response.data == {"code": "invalid-code"}
-
-
-@pytest.mark.django_db
-def test_validate_code_expired(api_client):
-    """Test validating an expired code."""
-    user = UserFactory()
-    ActivationCodeFactory(code="EXPIRED123456789", expires_at=timezone.now() - timedelta(days=1))
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "EXPIRED123456789"})
-
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-
-@pytest.mark.django_db
-def test_validate_code_max_uses_reached(api_client):
-    """Test validating a code that has reached max uses."""
-    user = UserFactory()
-    ActivationCodeFactory(code="MAXUSED123456789", max_uses=1, current_uses=1)
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "MAXUSED123456789"})
-
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-
-@pytest.mark.django_db
-def test_validate_code_multi_use(api_client):
-    """Test using a multi-use code with multiple users."""
-    multi_use_activation_code = ActivationCodeFactory(
-        code="MULTIUSE12345678",
-        max_uses=3,
-    )
-    users = []
-    for i in range(3):
-        user = UserFactory(email=f"user{i}@example.com")
-        users.append(user)
-
-    for i, user in enumerate(users):
-        api_client.force_authenticate(user=user)
-        response = api_client.post("/api/v1.0/activation/validate/", {"code": "MULTIUSE12345678"})
-
-        assert response.status_code == status.HTTP_201_CREATED
-        multi_use_activation_code.refresh_from_db()
-        assert multi_use_activation_code.current_uses == i + 1
-
-
-@pytest.mark.django_db
-def test_validate_code_unlimited_use(api_client):
-    """Test using an unlimited code with multiple users."""
-    unlimited_activation_code = ActivationCodeFactory(
-        code="UNLIMITED123CODE",
-        max_uses=0,  # Unlimited uses
-    )
-    for i in range(10):
-        user = UserFactory(email=f"user{i}@example.com")
-        api_client.force_authenticate(user=user)
-        response = api_client.post("/api/v1.0/activation/validate/", {"code": "UNLIMITED123CODE"})
-
-        assert response.status_code == status.HTTP_201_CREATED
-
-    # Code should still be valid
-    unlimited_activation_code.refresh_from_db()
-    assert unlimited_activation_code.is_valid() is True
-
-
-@pytest.mark.django_db
-def test_validate_code_logging_on_validation_error(api_client):
-    """Test that validation errors are logged."""
-    user = UserFactory()
-    api_client.force_authenticate(user=user)
-
-    # Create a code that will cause validation error
-    code = ActivationCodeFactory(
-        code="WILLEXPIRE123456", expires_at=timezone.now() + timedelta(days=1)
-    )
-
-    # Make it expire
-    code.expires_at = timezone.now() - timedelta(seconds=1)
-    code.save()
-
-    with patch("activation_codes.viewsets.logger"):
-        response = api_client.post("/api/v1.0/activation/validate/", {"code": "WILLEXPIRE123456"})
-
-    assert response.status_code == status.HTTP_400_BAD_REQUEST
-
-    # Note: In this case the serializer will catch it first
-    # so the warning might not be called, but this tests the flow
-
-
-@pytest.mark.django_db
-def test_unauthenticated_register_email(api_client):
-    """Test that unauthenticated users cannot register email."""
-    response = api_client.post(
-        "/api/v1.0/activation/register/",
-        {
-            "email": "test@example.com",
-        },
-    )
-    assert response.status_code == status.HTTP_401_UNAUTHORIZED
-
-
-@pytest.mark.django_db
-def test_register_email_success(api_client):
-    """Test successfully registering an email."""
-    user = UserFactory()
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post(
-        "/api/v1.0/activation/register/",
-        {},
-    )
-    assert response.status_code == status.HTTP_201_CREATED
-    assert response.data["code"] == "registration-successful"
-
-    registration = UserRegistrationRequest.objects.get(user=user)
-    assert registration.user == user
-
-
-@pytest.mark.django_db
-def test_register_already_created(api_client):
-    """Test successfully registering an email."""
-    user = UserFactory()
-    _registration = UserRegistrationRequest.objects.create(
-        user=user,
-    )
-
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post(
-        "/api/v1.0/activation/register/",
-    )
-    assert response.status_code == status.HTTP_200_OK
-    assert response.data == {"code": "registration-successful"}
-
-    assert UserRegistrationRequest.objects.filter(user=user).count() == 1
-
-
-@pytest.mark.django_db
-def test_validate_code_registered_user(api_client):
-    """Test validating a code for a user with a pre-existing registration."""
-    user = UserFactory()
-    _registration = UserRegistrationRequest.objects.create(
-        user=user,
-    )
-    activation_code = ActivationCodeFactory(code="TEST1234ABCD5678")
-
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post("/api/v1.0/activation/validate/", {"code": "TEST1234ABCD5678"})
-
-    assert response.status_code == status.HTTP_201_CREATED
-
-    _registration.refresh_from_db()
-    assert _registration.user_activation.activation_code == activation_code
-
-
-@responses.activate
-@pytest.mark.django_db
-def test_register_email_success_brevo(api_client, settings):
-    """Test successfully registering an email and notify Brevo."""
-    settings.BREVO_API_KEY = "test_brevo_api_key"
-    settings.BREVO_WAITING_LIST_ID = "test_waiting_list_id"
-
-    brevo_create_contact = responses.post(
-        "https://api.brevo.com/v3/contacts",
-        status=200,
-    )
-
-    brevo_mock = responses.post(
-        "https://api.brevo.com/v3/contacts/lists/test_waiting_list_id/contacts/add",
-        json={"message": "Contacts added successfully"},
-        status=201,
-    )
-
-    user = UserFactory()
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post(
-        "/api/v1.0/activation/register/",
-        {},
-    )
-    assert response.status_code == status.HTTP_201_CREATED
-    assert response.data["code"] == "registration-successful"
-
-    registration = UserRegistrationRequest.objects.get(user=user)
-    assert registration.user == user
-
-    assert len(brevo_create_contact.calls) == 1
-    assert brevo_create_contact.calls[0].request.headers["api-key"] == "test_brevo_api_key"
-    assert json.loads(brevo_create_contact.calls[0].request.body) == {
-        "email": user.email,
-        "updateEnabled": True,
-    }
-
-    assert len(brevo_mock.calls) == 1
-    assert brevo_mock.calls[0].request.headers["api-key"] == "test_brevo_api_key"
-    assert json.loads(brevo_mock.calls[0].request.body) == {"emails": [user.email]}
-
-    # Register again to test idempotency
-    response = api_client.post(
-        "/api/v1.0/activation/register/",
-        {},
-    )
-    assert response.status_code == status.HTTP_200_OK
-    assert response.data["code"] == "registration-successful"
-
-    assert len(brevo_mock.calls) == 1  # No new call made
-
-
-@responses.activate
-@pytest.mark.django_db
-def test_register_email_success_brevo_fails(api_client, settings):
-    """Test successfully registering an email, even if Brevo fails."""
-    settings.BREVO_API_KEY = "test_brevo_api_key"
-    settings.BREVO_WAITING_LIST_ID = "test_waiting_list_id"
-
-    _brevo_create_contact = responses.post(
-        "https://api.brevo.com/v3/contacts",
-        status=200,
-    )
-
-    brevo_mock = responses.post(
-        "https://api.brevo.com/v3/contacts/lists/test_waiting_list_id/contacts/add",
-        status=400,
-    )
-
-    user = UserFactory()
-    api_client.force_authenticate(user=user)
-
-    response = api_client.post(
-        "/api/v1.0/activation/register/",
-        {},
-    )
-    assert response.status_code == status.HTTP_201_CREATED
-    assert response.data["code"] == "registration-successful"
-
-    registration = UserRegistrationRequest.objects.get(user=user)
-    assert registration.user == user
-
-    assert len(brevo_mock.calls) == 1

src/backend/activation_codes/viewsets.py

@@ -1,153 +0,0 @@
-"""API ViewSets for activation codes."""
-
-import logging
-
-from django.conf import settings
-from django.core.exceptions import ValidationError
-from django.utils.translation import gettext_lazy as _
-
-from rest_framework import status, viewsets
-from rest_framework.decorators import action
-from rest_framework.response import Response
-
-from core.brevo import add_user_to_brevo_list
-from core.permissions import IsAuthenticated
-
-from . import models, serializers
-from .exceptions import InvalidCodeError, UserAlreadyActivatedError
-
-logger = logging.getLogger(__name__)
-
-
-class ActivationViewSet(viewsets.GenericViewSet):
-    """
-    ViewSet for handling user activation with codes.
-
-    Endpoints:
-    - GET /activation/status/ - Check if current user is activated
-    - POST /activation/validate/ - Validate and use an activation code
-    - POST /activation/register/ - Register an email to be notified later
-    """
-
-    permission_classes = [IsAuthenticated]
-    serializer_class = serializers.ActivationCodeValidationSerializer
-
-    @action(detail=False, methods=["get"], url_path="status")
-    def status(self, request):
-        """
-        Get the activation status of the current user.
-
-        Returns:
-            - is_activated: Whether the user has activated their account
-            - activation: Details of the activation (if exists)
-            - requires_activation: Whether activation is required by the system
-        """
-        requires_activation = getattr(settings, "ACTIVATION_REQUIRED", False)
-
-        try:
-            activation = models.UserActivation.objects.select_related("activation_code").get(
-                user=request.user
-            )
-            is_activated = True
-        except models.UserActivation.DoesNotExist:
-            activation = None
-            is_activated = False
-
-        response_data = {
-            "is_activated": is_activated,
-            "activation": activation,
-            "requires_activation": requires_activation,
-        }
-
-        return Response(
-            serializers.ActivationStatusSerializer(response_data).data, status=status.HTTP_200_OK
-        )
-
-    @action(detail=False, methods=["post"], url_path="validate")
-    def validate_code(self, request):
-        """
-        Validate an activation code and activate the user's account.
-
-        Request body:
-            - code: The activation code to validate
-
-        Returns:
-            - Success: Activation details
-            - Error: Validation error message
-        """
-        serializer = self.get_serializer(data=request.data)
-        serializer.is_valid(raise_exception=True)
-
-        code_value = serializer.validated_data["code"]
-
-        # Get the activation code
-        try:
-            activation_code = models.ActivationCode.objects.get(code=code_value)
-        except models.ActivationCode.DoesNotExist:
-            logger.info("Activation code %s does not exist", code_value)
-            return Response({"code": "invalid-code"}, status=status.HTTP_400_BAD_REQUEST)
-
-        # Use the code
-        try:
-            activation = activation_code.use(request.user)
-        except InvalidCodeError as exc:
-            logger.warning(exc)
-            return Response({"code": "invalid-code"}, status=status.HTTP_400_BAD_REQUEST)
-        except UserAlreadyActivatedError as exc:
-            logger.info(exc)
-            return Response(
-                {"code": "account-already-activated"}, status=status.HTTP_400_BAD_REQUEST
-            )
-
-        logger.info("User %s activated account with code %s", request.user.id, activation_code.code)
-
-        return Response(
-            {
-                "code": "activation-successful",
-                "detail": _("Your account has been successfully activated"),
-                "activation": serializers.UserActivationSerializer(activation).data,
-            },
-            status=status.HTTP_201_CREATED,
-        )
-
-    @action(detail=False, methods=["post"], url_path="register")
-    def register_email(self, request):
-        """
-        Register an email to be notified when activation codes are available.
-
-        Request body:
-            - email: The email address to register
-
-        Returns:
-            - Success: Confirmation message
-            - Error: Validation error message
-        """
-        serializer = serializers.UserRegistrationRequestSerializer(
-            data={},
-            context={"request": request},
-        )
-        serializer.is_valid(raise_exception=True)
-
-        # Create the registration
-        try:
-            serializer.save()
-        except ValidationError:
-            # user is already registered, it's OK
-            return Response(
-                {"code": "registration-successful"},
-                status=status.HTTP_200_OK,
-            )
-
-        add_user_to_brevo_list(
-            [serializer.validated_data["user"].email], settings.BREVO_WAITING_LIST_ID
-        )
-
-        logger.info(
-            "Registered email %s for activation notifications",
-            serializer.validated_data["user"].email,
-        )
-
-        return Response(
-            {"code": "registration-successful"},
-            status=status.HTTP_201_CREATED,
-        )

src/backend/chat/admin.py

@@ -9,11 +9,8 @@ from . import models
 class ChatConversationAdmin(admin.ModelAdmin):
     """Admin class for the ChatConversation model"""
 
-    autocomplete_fields = ("owner",)
-
     list_display = (
         "id",
-        "title",
         "created_at",
         "updated_at",
     )

src/backend/chat/agent_rag/albert_api_constants.py

@@ -1,150 +0,0 @@
-"""Constants and schemas for the Albert RAG agent from Albert API codebase."""
-
-from enum import Enum
-from typing import Annotated, Any, Dict, List, Literal, Optional, Self
-
-from pydantic import BaseModel, Field, StringConstraints, model_validator
-
-
-# - app/schemas/chunks.py
-class Chunk(BaseModel):
-    """Model representing a chunk of text with metadata."""
-
-    object: Literal["chunk"] = "chunk"
-    id: int
-    metadata: Dict[str, Any]
-    content: str
-
-
-class Chunks(BaseModel):
-    """Model representing a list of chunks."""
-
-    object: Literal["list"] = "list"
-    data: List[Chunk]
-
-
-# - app/schemas/usage.py
-class CarbonFootprintUsageKWh(BaseModel):
-    """Model representing the carbon footprint usage in kWh (kilowatt-hours)."""
-
-    min: Optional[float] = Field(default=None, description="Minimum carbon footprint in kWh.")
-    max: Optional[float] = Field(default=None, description="Maximum carbon footprint in kWh.")
-
-
-class CarbonFootprintUsageKgCO2eq(BaseModel):
-    """Model representing the carbon footprint usage in kgCO2eq (kilograms of CO2 equivalent)."""
-
-    min: Optional[float] = Field(
-        default=None, description="Minimum carbon footprint in kgCO2eq (global warming potential)."
-    )
-    max: Optional[float] = Field(
-        default=None, description="Maximum carbon footprint in kgCO2eq (global warming potential)."
-    )
-
-
-class CarbonFootprintUsage(BaseModel):
-    """Model representing the carbon footprint usage in kWh and kgCO2eq."""
-
-    kWh: CarbonFootprintUsageKWh = Field(default_factory=CarbonFootprintUsageKWh)
-    kgCO2eq: CarbonFootprintUsageKgCO2eq = Field(default_factory=CarbonFootprintUsageKgCO2eq)
-
-
-class BaseUsage(BaseModel):
-    """Base model for usage statistics in the Albert API."""
-
-    prompt_tokens: int = Field(
-        default=0, description="Number of prompt tokens (e.g. input tokens)."
-    )
-    completion_tokens: int = Field(
-        default=0, description="Number of completion tokens (e.g. output tokens)."
-    )
-    total_tokens: int = Field(
-        default=0, description="Total number of tokens (e.g. input and output tokens)."
-    )
-    cost: float = Field(default=0.0, description="Total cost of the request.")
-    carbon: CarbonFootprintUsage = Field(default_factory=CarbonFootprintUsage)
-
-
-# - app/schemas/usage.py
-class Detail(BaseModel):
-    """Model representing a detail in the usage statistics."""
-
-    id: str
-    model: str
-    usage: BaseUsage = Field(default_factory=BaseUsage)
-
-
-class Usage(BaseUsage):
-    """Model representing the usage statistics for the Albert API."""
-
-    details: List[Detail] = []
-
-
-class SearchMethod(str, Enum):
-    """
-    Enum representing the search methods available (will be displayed in this order in playground).
-    """
-
-    MULTIAGENT = "multiagent"
-    HYBRID = "hybrid"
-    SEMANTIC = "semantic"
-    LEXICAL = "lexical"
-
-
-class SearchArgs(BaseModel):
-    """Model representing the arguments for a search request in the Albert API."""
-
-    collections: List[Any] = Field(default=[], description="List of collections ID")
-    rff_k: int = Field(default=20, description="k constant in RFF algorithm")
-    k: int = Field(gt=0, default=4, description="Number of results to return")
-    method: SearchMethod = Field(default=SearchMethod.SEMANTIC)
-    score_threshold: Optional[float] = Field(
-        default=0.0,
-        ge=0.0,
-        le=1.0,
-        description=(
-            "Score of cosine similarity threshold for filtering results, "
-            "only available for semantic search method."
-        ),
-    )
-    web_search: bool = Field(
-        default=False, description="Whether add internet search to the results."
-    )
-    web_search_k: int = Field(default=5, description="Number of results to return for web search.")
-
-    @model_validator(mode="after")
-    def score_threshold_filter(self) -> Self:
-        """Validate the score threshold based on the search method."""
-        if self.score_threshold and self.method not in (
-            SearchMethod.SEMANTIC,
-            SearchMethod.MULTIAGENT,
-        ):
-            raise ValueError(
-                "Score threshold is only available for semantic and multiagent search methods."
-            )
-        return self
-
-
-class SearchRequest(SearchArgs):
-    """Model representing a search request in the Albert API."""
-
-    prompt: Annotated[
-        str,
-        StringConstraints(strip_whitespace=True, min_length=1),
-    ] = Field(description="Prompt related to the search")
-
-
-class Search(BaseModel):
-    """Model representing a search result in the Albert API."""
-
-    method: SearchMethod
-    score: float
-    chunk: Chunk
-
-
-class Searches(BaseModel):
-    """Model representing a list of search results in the Albert API."""
-
-    object: Literal["list"] = "list"
-    data: List[Search]
-    usage: Usage = Field(default_factory=Usage, description="Usage information for the request.")

src/backend/chat/agent_rag/constants.py

@@ -1,41 +0,0 @@
-"""Constants for RAG (Retrieval-Augmented Generation) results."""
-
-from typing import List
-
-from pydantic import BaseModel, Field
-
-
-class RAGWebUsage(BaseModel):
-    """
-    Model representing the usage statistics for web results in RAG (Retrieval-Augmented Generation).
-    """
-
-    prompt_tokens: int = Field(default=0, description="Number of prompt tokens used.")
-    completion_tokens: int = Field(default=0, description="Number of completion tokens generated.")
-
-
-class RAGWebResult(BaseModel):
-    """Model representing a single web result in RAG (Retrieval-Augmented Generation)."""
-
-    url: str = Field(..., description="URL of the web result.")
-    content: str = Field(..., description="Content of the web result chunk.")
-    score: float = Field(
-        ..., description="Relevance score of the web result, typically between 0 and 1."
-    )
-
-
-class RAGWebResults(BaseModel):
-    """Model representing a list of web results in RAG (Retrieval-Augmented Generation)."""
-
-    data: List[RAGWebResult]
-    usage: RAGWebUsage = Field(..., description="RAG usage statistics.")
-
-    def to_prompt(self) -> str:
-        """Convert the web results to a prompt string."""
-        _format = " - From: {url}:\n   content: {content}\n\n"
-        return (
-            "\n\n".join(
-                _format.format(url=result.url, content=result.content) for result in self.data
-            )
-            + "\n\n"
-        )

src/backend/chat/agent_rag/document_converter/markitdown.py

@@ -1,43 +0,0 @@
-"""Document Converter using MarkItDown"""
-
-import os.path
-from io import BytesIO
-
-from markitdown import MarkItDown
-
-
-class DocumentConverter:
-    """Simple document converter that uses MarkItDown to convert documents to Markdown format."""
-
-    def __init__(self):
-        """Initialize the DocumentConverter with MarkItDown."""
-        self.converter = MarkItDown()
-
-    def convert_raw(  # pylint: disable=unused-argument
-        self,
-        *,
-        name: str,
-        content_type: str,
-        content: bytes,
-    ) -> str:
-        """
-        Convert a document to Markdown format.
-        The name, content_type, and content parameters comes from the user input
-        (vercel SDK Attachment, or BinaryContent).
-
-        Args:
-            name (str): The name of the document.
-            content_type (str): The MIME type of the document (e.g., "application/pdf").
-            content (bytes): The content of the document as bytes.
-        """
-        return self._convert(BytesIO(content), file_extension=os.path.splitext(name)[1])
-
-    def _convert(self, document: BytesIO, file_extension: str) -> str:
-        """
-        Convert the given document using the underlying DocumentConverter.
-        """
-        conversion = self.converter.convert_stream(
-            document, file_extension=file_extension or ".txt"
-        )
-        document_markdown = conversion.text_content
-        return document_markdown

src/backend/chat/agent_rag/document_rag_backends/albert_rag_backend.py

@@ -1,190 +0,0 @@
-"""Implementation of the Albert API for RAG document search."""
-
-import json
-import logging
-from io import BytesIO
-from typing import Optional
-from urllib.parse import urljoin
-
-from django.conf import settings
-
-import requests
-
-from chat.agent_rag.albert_api_constants import Searches
-from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
-from chat.agent_rag.document_converter.markitdown import DocumentConverter
-from chat.agent_rag.document_rag_backends.base_rag_backend import BaseRagBackend
-
-logger = logging.getLogger(__name__)
-
-
-class AlbertRagBackend(BaseRagBackend):  # pylint: disable=too-many-instance-attributes
-    """
-    This class is a placeholder for the Albert API implementation.
-    It is designed to be used with the RAG (Retrieval-Augmented Generation) document search system.
-
-    It provides methods to:
-    - Create a collection for the search operation.
-    - Parse documents and convert them to Markdown format:
-       + Handle PDF parsing using the Albert API.
-       + Use the DocumentConverter (markitdown) for other formats.
-    - Store parsed documents in the Albert collection.
-    - Perform a search operation using the Albert API.
-    """
-
-    def __init__(self, collection_id: Optional[str] = None):
-        # Initialize any necessary parameters or configurations here
-        super().__init__(collection_id)
-        self._base_url = settings.ALBERT_API_URL
-        self._headers = {
-            "Authorization": f"Bearer {settings.ALBERT_API_KEY}",
-        }
-        self._collections_endpoint = urljoin(self._base_url, "/v1/collections")
-        self._documents_endpoint = urljoin(self._base_url, "/v1/documents")
-        self._pdf_parser_endpoint = urljoin(self._base_url, "/v1/parse-beta")
-        self._search_endpoint = urljoin(self._base_url, "/v1/search")
-
-        self._default_collection_description = "Temporary collection for RAG document search"
-
-    def create_collection(self, name: str, description: Optional[str] = None) -> str:
-        """
-        Create a temporary collection for the search operation.
-        This method should handle the logic to create or retrieve an existing collection.
-        """
-        response = requests.post(
-            self._collections_endpoint,
-            headers=self._headers,
-            json={
-                "name": name,
-                "description": description or self._default_collection_description,
-                "visibility": "private",
-            },
-            timeout=settings.ALBERT_API_TIMEOUT,
-        )
-        response.raise_for_status()
-        self.collection_id = str(response.json()["id"])
-        return self.collection_id
-
-    def delete_collection(self) -> None:
-        """
-        Delete the current collection
-        """
-        response = requests.delete(
-            urljoin(f"{self._collections_endpoint}/", self.collection_id),
-            headers=self._headers,
-            timeout=settings.ALBERT_API_TIMEOUT,
-        )
-        response.raise_for_status()
-
-    def parse_pdf_document(self, name: str, content_type: str, content: BytesIO) -> str:
-        """
-        Parse the PDF document content and return the text content.
-        This method should handle the logic to convert the PDF into
-        a format suitable for the Albert API.
-        """
-        response = requests.post(
-            self._pdf_parser_endpoint,
-            headers=self._headers,
-            files={
-                "file": (
-                    name,
-                    content,
-                    content_type,
-                ),  # Use the name as the filename in the request
-                "output_format": (None, "markdown"),  # Specify the output format as Markdown,
-            },
-            timeout=settings.ALBERT_API_PARSE_TIMEOUT,
-        )
-        response.raise_for_status()
-
-        return "\n\n".join(
-            document_page["content"] for document_page in response.json().get("data", [])
-        )
-
-    def parse_document(self, name: str, content_type: str, content: BytesIO):
-        """
-        Parse the document and prepare it for the search operation.
-        This method should handle the logic to convert the document
-        into a format suitable for the Albert API.
-
-        Args:
-            name (str): The name of the document.
-            content_type (str): The MIME type of the document (e.g., "application/pdf").
-            content (BytesIO): The content of the document as a BytesIO stream.
-
-        Returns:
-            str: The document content in Markdown format.
-        """
-        # Implement the parsing logic here
-        if content_type == "application/pdf":
-            # Handle PDF parsing
-            markdown_content = self.parse_pdf_document(
-                name=name, content_type=content_type, content=content
-            )
-        else:
-            markdown_content = DocumentConverter().convert_raw(
-                name=name, content_type=content_type, content=content
-            )
-
-        return markdown_content
-
-    def store_document(self, name: str, content: str) -> None:
-        """
-        Store the document content in the Albert collection.
-        This method should handle the logic to send the document content to the Albert API.
-
-        Args:
-            name (str): The name of the document.
-            content (str): The content of the document in Markdown format.
-        """
-        response = requests.post(
-            urljoin(self._base_url, self._documents_endpoint),
-            headers=self._headers,
-            files={
-                "file": (f"{name}.md", BytesIO(content.encode("utf-8")), "text/markdown"),
-                "collection": (None, int(self.collection_id)),
-                "metadata": (None, json.dumps({"document_name": name})),  # undocumented API
-            },
-            timeout=settings.ALBERT_API_TIMEOUT,
-        )
-        logger.debug(response.json())
-        response.raise_for_status()
-
-    def search(self, query) -> RAGWebResults:
-        """
-        Perform a search using the Albert API based on the provided query.
-
-        Args:
-            query (str): The search query.
-
-        Returns:
-            RAGWebResults: The search results.
-        """
-        response = requests.post(
-            urljoin(self._base_url, self._search_endpoint),
-            headers=self._headers,
-            json={
-                "collections": [int(self.collection_id)],
-                "prompt": query,
-                "score_threshold": 0.6,
-            },
-            timeout=settings.ALBERT_API_TIMEOUT,
-        )
-        response.raise_for_status()
-
-        searches = Searches(**response.json())
-
-        return RAGWebResults(
-            data=[
-                RAGWebResult(
-                    url=result.chunk.metadata["document_name"],
-                    content=result.chunk.content,
-                    score=result.score,
-                )
-                for result in searches.data
-            ],
-            usage=RAGWebUsage(
-                prompt_tokens=searches.usage.prompt_tokens,
-                completion_tokens=searches.usage.completion_tokens,
-            ),
-        )

src/backend/chat/agent_rag/document_rag_backends/base_rag_backend.py

@@ -1,94 +0,0 @@
-"""Implementation of the Albert API for RAG document search."""
-
-import logging
-from contextlib import contextmanager
-from io import BytesIO
-from typing import Optional
-
-from chat.agent_rag.constants import RAGWebResults
-
-logger = logging.getLogger(__name__)
-
-
-class BaseRagBackend:
-    """Base class for RAG backends."""
-
-    def __init__(self, collection_id: Optional[str] = None):
-        """Backend settings."""
-        self.collection_id = collection_id
-        self._default_collection_description = "Temporary collection for RAG document search"
-
-    def create_collection(self, name: str, description: Optional[str] = None) -> str:
-        """
-        Create a temporary collection for the search operation.
-        This method should handle the logic to create or retrieve an existing collection.
-        """
-        raise NotImplementedError("Must be implemented in subclass.")
-
-    def parse_document(self, name: str, content_type: str, content: BytesIO):
-        """
-        Parse the document and prepare it for the search operation.
-        This method should handle the logic to convert the document
-        into a format suitable for the Albert API.
-
-        Args:
-            name (str): The name of the document.
-            content_type (str): The MIME type of the document (e.g., "application/pdf").
-            content (BytesIO): The content of the document as a BytesIO stream.
-
-        Returns:
-            str: The document content in Markdown format.
-        """
-        raise NotImplementedError("Must be implemented in subclass.")
-
-    def store_document(self, name: str, content: str) -> None:
-        """
-        Store the document content in the Albert collection.
-        This method should handle the logic to send the document content to the Albert API.
-
-        Args:
-            name (str): The name of the document.
-            content (str): The content of the document in Markdown format.
-        """
-        raise NotImplementedError("Must be implemented in subclass.")
-
-    def parse_and_store_document(self, name: str, content_type: str, content: BytesIO) -> str:
-        """
-        Parse the document and store it in the Albert collection.
-
-        Args:
-            name (str): The name of the document.
-            content_type (str): The MIME type of the document (e.g., "application/pdf").
-            content (BytesIO): The content of the document as a BytesIO stream.
-        """
-        if not self.collection_id:
-            raise RuntimeError("The RAG backend requires collection_id")
-
-        document_content = self.parse_document(name, content_type, content)
-        self.store_document(name, document_content)
-        return document_content
-
-    def delete_collection(self) -> None:
-        """
-        Delete the collection.
-        This method should handle the logic to delete the collection from the backend.
-        """
-        raise NotImplementedError("Must be implemented in subclass.")
-
-    def search(self, query) -> RAGWebResults:
-        """
-        Search the collection for the given query.
-        """
-        raise NotImplementedError("Must be implemented in subclass.")
-
-    @classmethod
-    @contextmanager
-    def temporary_collection(cls, name: str, description: Optional[str] = None):
-        """Context manager for RAG backend with temporary collections."""
-        backend = cls()
-
-        backend.create_collection(name=name, description=description)
-        try:
-            yield backend
-        finally:
-            backend.delete_collection()

src/backend/chat/agent_rag/document_search/albert_api.py

@@ -1,208 +0,0 @@
-"""Implementation of the Albert API for RAG document search."""
-
-import json
-import logging
-from io import BytesIO
-from urllib.parse import urljoin
-
-from django.conf import settings
-
-import requests
-
-from chat.agent_rag.albert_api_constants import Searches
-from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
-from chat.agent_rag.document_converter.markitdown import DocumentConverter
-from chat.models import ChatConversation
-
-logger = logging.getLogger(__name__)
-
-
-class AlbertRagDocumentSearch:
-    """
-    This class is a placeholder for the Albert API implementation.
-    It is designed to be used with the RAG (Retrieval-Augmented Generation) document search system.
-
-    It provides methods to:
-    - Create a collection for the search operation.
-    - Parse documents and convert them to Markdown format:
-       + Handle PDF parsing using the Albert API.
-       + Use the DocumentConverter (markitdown) for other formats.
-    - Store parsed documents in the Albert collection.
-    - Perform a search operation using the Albert API.
-    """
-
-    def __init__(self, conversation: ChatConversation):
-        # Initialize any necessary parameters or configurations here
-        self._base_url = settings.ALBERT_API_URL
-        self._headers = {
-            "Authorization": f"Bearer {settings.ALBERT_API_KEY}",
-        }
-        self._collections_endpoint = urljoin(self._base_url, "/v1/collections")
-        self._documents_endpoint = urljoin(self._base_url, "/v1/documents")
-        self._pdf_parser_endpoint = urljoin(self._base_url, "/v1/parse-beta")
-        self._search_endpoint = urljoin(self._base_url, "/v1/search")
-
-        self.conversation = conversation
-
-    @property
-    def _albert_collection_id(self):
-        """
-        Generate the collection name based on the conversation ID.
-        This is used to create or retrieve a collection for the search operation.
-        """
-        return f"conversation-{self.conversation.pk}"
-
-    @property
-    def collection_id(self) -> int:
-        """
-        Get the collection ID for the current conversation.
-
-        Might be created later by self._create_collection() if it does not exist.
-        """
-        return int(self.conversation.collection_id) if self.conversation.collection_id else None
-
-    def _create_collection(self) -> bool:
-        """
-        Create a temporary collection for the search operation.
-        This method should handle the logic to create or retrieve an existing collection.
-        """
-        response = requests.post(
-            self._collections_endpoint,
-            headers=self._headers,
-            json={
-                "name": self._albert_collection_id,
-                "description": "Temporary collection for RAG document search",
-                "visibility": "private",
-            },
-            timeout=settings.ALBERT_API_TIMEOUT,
-        )
-        response.raise_for_status()
-        self.conversation.collection_id = str(response.json()["id"])
-        return True
-
-    def _parse_pdf_document(self, name: str, content_type: str, content: BytesIO) -> str:
-        """
-        Parse the PDF document content and return the text content.
-        This method should handle the logic to convert the PDF into
-        a format suitable for the Albert API.
-        """
-        response = requests.post(
-            self._pdf_parser_endpoint,
-            headers=self._headers,
-            files={
-                "file": (
-                    name,
-                    content,
-                    content_type,
-                ),  # Use the name as the filename in the request
-                "output_format": (None, "markdown"),  # Specify the output format as Markdown,
-            },
-            timeout=settings.ALBERT_API_PARSE_TIMEOUT,
-        )
-        response.raise_for_status()
-
-        return "\n\n".join(
-            document_page["content"] for document_page in response.json().get("data", [])
-        )
-
-    def parse_document(self, name: str, content_type: str, content: BytesIO):
-        """
-        Parse the document and prepare it for the search operation.
-        This method should handle the logic to convert the document
-        into a format suitable for the Albert API.
-
-        Args:
-            name (str): The name of the document.
-            content_type (str): The MIME type of the document (e.g., "application/pdf").
-            content (BytesIO): The content of the document as a BytesIO stream.
-
-        Returns:
-            str: The document content in Markdown format.
-        """
-        # Implement the parsing logic here
-        if content_type == "application/pdf":
-            # Handle PDF parsing
-            markdown_content = self._parse_pdf_document(
-                name=name, content_type=content_type, content=content
-            )
-        else:
-            markdown_content = DocumentConverter().convert_raw(
-                name=name, content_type=content_type, content=content
-            )
-
-        return markdown_content
-
-    def _store_document(self, name: str, content: str):
-        """
-        Store the document content in the Albert collection.
-        This method should handle the logic to send the document content to the Albert API.
-
-        Args:
-            content (str): The content of the document in Markdown format.
-        """
-        if not self.collection_id and not self._create_collection():
-            raise RuntimeError("Failed to create or retrieve the collection.")
-
-        response = requests.post(
-            urljoin(self._base_url, self._documents_endpoint),
-            headers=self._headers,
-            files={
-                "file": (f"{name}.md", BytesIO(content.encode("utf-8")), "text/markdown"),
-                "collection": (None, int(self.collection_id)),
-                "metadata": (None, json.dumps({"document_name": name})),  # undocumented API
-            },
-            timeout=settings.ALBERT_API_TIMEOUT,
-        )
-        logger.debug(response.json())
-        response.raise_for_status()
-
-    def parse_and_store_document(self, name: str, content_type: str, content: BytesIO):
-        """
-        Parse the document and store it in the Albert collection.
-
-        Args:
-            name (str): The name of the document.
-            content_type (str): The MIME type of the document (e.g., "application/pdf").
-            content (BytesIO): The content of the document as a BytesIO stream.
-        """
-        document_content = self.parse_document(name, content_type, content)
-        self._store_document(name, document_content)
-        return document_content
-
-    def search(self, query, results_count: int = 4) -> RAGWebResults:
-        """
-        Perform a search using the Albert API based on the provided query.
-
-        :param query: The search query string.
-        :param results_count: The number of results to return.
-        :return: Search results from the Albert API.
-        """
-        response = requests.post(
-            urljoin(self._base_url, self._search_endpoint),
-            headers=self._headers,
-            json={
-                "collections": [self.collection_id],
-                "prompt": query,
-                "score_threshold": 0.6,
-                "k": results_count,  # Number of chunks to return from the search
-            },
-            timeout=settings.ALBERT_API_TIMEOUT,
-        )
-        response.raise_for_status()
-
-        searches = Searches(**response.json())
-
-        return RAGWebResults(
-            data=[
-                RAGWebResult(
-                    url=result.chunk.metadata["document_name"],
-                    content=result.chunk.content,
-                    score=result.score,
-                )
-                for result in searches.data
-            ],
-            usage=RAGWebUsage(
-                prompt_tokens=searches.usage.prompt_tokens,
-                completion_tokens=searches.usage.completion_tokens,
-            ),
-        )

src/backend/chat/agent_rag/web_search/albert_api.py

@@ -1,113 +0,0 @@
-"""
-Albert API web search manager.
-
-Instead of developing a full web search flow, we simply use the Albert API /v1/search endpoint.
-
-See: https://albert.api.etalab.gouv.fr/documentation#tag/Search
-
-Under the hood, on Albert side:
- - It create a temporary collection
- - It performs a web search using the query
- - It returns the results as a list of URls
- - It loads and parses the content of each URL (vectorization)
-   and stores the results in the temporary collection
- - It makes a semanctic search on the temporary collection using the query
- - It returns the results as a list of chunks with metadata
- - It deletes the temporary collection
-"""
-
-import logging
-from urllib.parse import urljoin
-
-from django.conf import settings
-
-import requests
-
-from ..albert_api_constants import Searches, SearchRequest
-from ..constants import RAGWebResult, RAGWebResults, RAGWebUsage
-from .base import BaseWebSearchManager
-
-logger = logging.getLogger(__name__)
-
-
-class AlbertWebSearchManager(BaseWebSearchManager):
-    """
-    A class to manage web search operations using the Albert API.
-    """
-
-    def __init__(self):
-        """Initializes with the base URL and endpoints for the Albert API."""
-        self._base_url = settings.ALBERT_API_URL
-        self._headers = {
-            "Authorization": f"Bearer {settings.ALBERT_API_KEY}",
-            "Content-Type": "application/json",
-        }
-        self._search_endpoint = urljoin(self._base_url, "/v1/search")
-
-    @staticmethod
-    def _clean_url(url: str) -> str:
-        """
-        Clean the URL by removing the trailing '.html'.
-        We want it to fail when Albert fixes the bug that adds '.html' to the end of URLs.
-        Note: this is a bad workaround because when fixed it may break existing URLs.
-
-        Args:
-            url (str): The URL to clean.
-
-        Returns:
-            str: The cleaned URL.
-        """
-        return url.rsplit(".html", 1)[0]
-
-    def web_search(self, query: str) -> RAGWebResults:
-        """
-        Perform a web search using the Albert API.
-
-        Args:
-            query (str): The search query.
-
-        Returns:
-            Searches: A Searches object containing the search results.
-
-        Raises:
-            ValueError: If the query is empty.
-            requests.HTTPError: If the request to the Albert API fails.
-            requests.exceptions.JSONDecodeError: If the response body does not
-                contain valid json
-        """
-        if not query.strip():
-            raise ValueError("Search query cannot be empty.")
-
-        search_request = SearchRequest(
-            prompt=query,
-            web_search=True,  # Enable web search
-            web_search_k=settings.RAG_WEB_SEARCH_MAX_RESULTS,  # Number of web search results
-            k=settings.RAG_WEB_SEARCH_CHUNK_NUMBER,  # Number of chunks to return from the search
-        )
-
-        logger.debug("Albert API search request: %s", search_request.model_dump())
-
-        response = requests.post(
-            self._search_endpoint,
-            headers=self._headers,
-            json=search_request.model_dump(mode="json", exclude_unset=True),
-            timeout=settings.ALBERT_API_TIMEOUT,
-        )
-        response.raise_for_status()
-
-        searches = Searches(**response.json())
-
-        return RAGWebResults(
-            data=[
-                RAGWebResult(
-                    url=self._clean_url(result.chunk.metadata["document_name"]),
-                    content=result.chunk.content,
-                    score=result.score,
-                )
-                for result in searches.data
-            ],
-            usage=RAGWebUsage(
-                prompt_tokens=searches.usage.prompt_tokens,  # pylint: disable=no-member
-                completion_tokens=searches.usage.completion_tokens,  # pylint: disable=no-member
-            ),
-        )

src/backend/chat/agent_rag/web_search/base.py

@@ -1,24 +0,0 @@
-"""Base class for web search managers."""
-
-from ..constants import RAGWebResults
-
-
-class BaseWebSearchManager:
-    """
-    A class to manage web search operations.
-
-    This is an abstract base class that should be implemented
-    for specific web search managers.
-    """
-
-    def web_search(self, query: str) -> RAGWebResults:
-        """
-        Perform a web search.
-
-        Args:
-            query (str): The search query.
-
-        Returns:
-            RAGWebResults: A Searches object containing the search results.
-        """
-        raise NotImplementedError()

src/backend/chat/agent_rag/web_search/mocked.py

@@ -1,68 +0,0 @@
-"""Mocked web search manager for testing purposes."""
-
-# pylint: disable=line-too-long
-import logging
-
-from ..constants import RAGWebResult, RAGWebResults, RAGWebUsage
-
-logger = logging.getLogger(__name__)
-
-result_1 = {
-    "url": "https://www.lemonde.fr/sciences/article/2025/06/25/le-telescope-james-webb-decouvre-sa-premiere-exoplanete-identifiee-comme-une-petite-planete-froide_6615888_1650684.html",
-    "content": "+ [La beauté créatrice](https://la-beaute-creatrice.lemonde.fr/ \"La beauté créatrice\")\n\t+ [Perspectives](https://www.lemonde.fr/perspectives/ \"Perspectives\")    \n\t+ [Gestion des cookies](#)\n* [Recherche](https://www.lemonde.fr/recherche/)\n\n             Cet article vous est offert Pour lire gratuitement cet article réservé aux abonnés, connectez\\-vous [Se connecter](https://secure.lemonde.fr/sfuser/connexion?gift=true)  Vous n'êtes pas inscrit sur Le Monde ?   \n [Inscrivez\\-vous gratuitement](https://secure.lemonde.fr/sfuser/register?gift=true)        * [Sciences Sciences](https://www.lemonde.fr/sciences/)\n* [Astronomie Astronomie](https://www.lemonde.fr/cosmos/)\n\n  \n\nLe télescope James\\-Webb d��couvre sa première exoplanète, identifiée comme une petite planète froide\n====================================================================================================\n\n   L’observation a été faite grâce à une méthode prometteuse pour détecter des planètes d’une taille similaire à celles du système solaire.     Le Monde avec AFP \n\n  Publié le 25 juin 2025 à 17h49, modifié le 26 juin 2025 à 03h24   Temps de Lecture 2 min. \n\n     Lire plus tard     Partager  * Partager sur Messenger\n* Partager sur Facebook\n* Envoyer par e\\-mail\n* Partager sur Linkedin\n* Copier le lien\n\n         <img src='https://img.lemde.fr/2025/06/25/0/0/866/867/664/0/75/0/1173df3_upload-1-g41rmpa9cg0l-902429.jpg' alt='Une image du disque protoplanétaire autour de l’étoile TWA 7, enregistrée à l’aide de l’instrument Sphere du télescope basé au Chili, est présentée le 25\xa0juin 2025 avec une image superposée capturée par l’instrument MIRI du télescope spatial James-Webb.' title='' width='664' height='443' />   ![Une image du disque protoplanétaire autour de l’étoile TWA 7, enregistrée à l’aide de l’instrument Sphere du télescope basé au Chili, est présentée le 25\xa0juin 2025 avec une image superposée capturée par l’instrument MIRI du télescope spatial James-Webb.](https://img.lemde.fr/2025/06/25/0/0/866/867/664/0/75/0/1173df3_upload-1-g41rmpa9cg0l-902429.jpg)  \n\nUne image du disque protoplanétaire autour de l’étoile TWA 7, enregistrée à l’aide de l’instrument Sphere du télescope basé au Chili, est présentée le 25\xa0juin 2025 avec une image superposée capturée par l’instrument MIRI du télescope spatial James\\-Webb. WST/ESO/REUTERS \n\n   Le télescope spatial James\\-Webb (JWST) a découvert sa première exoplanète dans l’univers proche. Une observation faite grâce à une méthode prometteuse pour détecter des planètes d’une taille similaire à celles du système solaire.\n\n  Depuis 2022, à 1,5\xa0million de kilomètres de la Terre, le JWST a aidé à caractériser plusieurs exoplanètes. *«\xa0Il a passé énormément de temps à observer des planètes qui n’ont jamais été imagées\xa0»*, explique l’astrophysicienne Anne\\-Marie Lagrange, première autrice de l’étude sur le sujet, parue dans [*Nature*](https://www.nature.com/articles/s41550-024-02401-w?utm_source=chatgpt.com \"Nouvelle fenêtre\") mercredi 25\xa0juin.\n\n L’exercice est compliqué du fait que les exoplanètes *«\xa0sont très peu lumineuses parce qu’elles ne sont pas chaudes\xa0»*, mais aussi et surtout du fait qu’*«\xa0on est aveuglé par la lumière de l’étoile autour de laquelle elles tournent\xa0»*, ajoute cette chercheuse du CNRS au Laboratoire d’instrumentation et de recherche en astrophysique de l’Observatoire de Paris.\n\n La parade du James\\-Webb repose sur son coronographe, un instrument qui s’inspire du phénomène de l’éclipse solaire en masquant l’étoile pour mieux révéler ce qui l’entoure, et sur son spectrographe MIRI, capable d’imager les astres les plus discrets grâce à une vision infrarouge. Ses utilisateurs ont pointé le télescope vers l’étoile TWA 7, située dans notre galaxie à une centaine d’années\\-lumière de la Terre, autrement dit sa très petite banlieue. La cible, initialement détectée par le télescope Hubble, était prometteuse à double titre.",
-    "score": 0.51039016,
-}
-result_2 = {
-    "url": "https://www.lemonde.fr/sciences/article/2025/06/25/le-telescope-james-webb-decouvre-sa-premiere-exoplanete-identifiee-comme-une-petite-planete-froide_6615888_1650684.html",
-    "content": "Le télescope James\\-Webb découvre sa première exoplanète, identifiée comme une petite planète froide",
-    "score": 0.49020067,
-}
-result_3 = {
-    "url": "https://www.lemonde.fr/sciences/article/2025/06/25/le-telescope-james-webb-decouvre-sa-premiere-exoplanete-identifiee-comme-une-petite-planete-froide_6615888_1650684.html",
-    "content": "---------------\n\n D’abord, parce qu’elle est jeune de seulement 6,4\xa0millions d’années et donc très susceptible de voir se former des corps planétaires dans le disque de matière la ceinturant. Ensuite, parce que le télescope voit ce disque protoplanétaire par le dessus. Son observation avec l’instrument Sphere du Très Grand Télescope (VLT), situé au Chili, avait permis d’y distinguer trois anneaux s’étageant sur une distance allant jusqu’à plus de cent fois celle séparant la Terre du Soleil.\n\n  Et c’est dans la partie dégarnie du deuxième anneau que l’instrument du James\\-Webb a détecté une *«\xa0source\xa0»* lumineuse, baptisée TWA 7b. Ayant exclu que la découverte s’avère être un objet du système solaire ou une galaxie lointaine, les astronomes l’ont identifiée comme une petite planète froide, d’une masse dix fois inférieure à celles imagées jusqu’ici avec d’autres instruments. Ils estiment sa masse comparable à celle de Saturne, une planète gazeuse qui ne *«\xa0pèse\xa0»* que le tiers de Jupiter, géante gazeuse et poids lourd de notre système solaire.\n\n Avec le James\\-Webb, *«\xa0on est tombé d’un facteur dix en capacité de détection\xa0»*, explique Anne\\-Marie Lagrange, car les planètes les plus *«\xa0légères\xa0»* imagées jusqu’ici depuis le sol pesaient à peu près trois fois la masse de Jupiter. *«\xa0La plupart des autres exoplanètes imagées sont ce qu’on appelle des super\\-Jupiter\xa0»*, ayant de huit à douze fois la masse de cette dernière.\n\n  Lire aussi \\| Article réservé à nos abonnés [Potentielle présence de vie sur l’exoplanète K2\\-18b\xa0: anatomie d’un faux positif](https://www.lemonde.fr/sciences/article/2025/05/13/potentielle-presence-de-vie-sur-l-exoplanete-k2-18b-anatomie-d-un-faux-positif_6605670_1650684.html)      Lire plus tard    \n\nUn télescope prometteur attendu pour 2028",
-    "score": 0.48312354,
-}
-result_4 = {
-    "url": "https://www.lemonde.fr/sciences/article/2025/06/25/le-telescope-james-webb-decouvre-sa-premiere-exoplanete-identifiee-comme-une-petite-planete-froide_6615888_1650684.html",
-    "content": "-----------------------------------------\n\n La performance a d’autant plus d’intérêt que dans le bestiaire planétaire, les planètes rocheuses comme la Terre ou Mars ont des masses beaucoup plus faibles que les planètes gazeuses. Or ces exoplanètes rocheuses constituent une cible ultime des découvreurs de mondes potentiellement habitables.\n\n   Le Monde Guides d’achat   [Aspirateurs robots Les meilleurs aspirateurs robots Lire](https://www.lemonde.fr/guides-d-achat/article/2021/10/25/les-meilleurs-aspirateurs-robots_6099813_5306571.html)   Anne\\-Marie Lagrange souhaiterait désormais *«\xa0découvrir les planètes les plus légères et peut\\-être de trouver des Terres\xa0»*. Avant d’ajouter aussit��t que si *«\xa0on veut comprendre comment les systèmes planétaires se forment, il ne suffit pas de voir les planètes très ou pas massives\xa0»*. Car il faut pouvoir détecter tous les types de planètes, afin de déterminer in fine si notre système solaire est unique ou pas.\n\n Les astronomes estiment que le JWST a le potentiel de détecter et d’imager des planètes ayant une masse encore plus faible que TWA 7b. Mais il faudra de futurs instruments, comme le Télescope extrêmement large (ELT) attendu pour 2028, pour espérer saisir l’image de mondes d’une taille similaire au nôtre.\n\n  Lire aussi \\| Article réservé à nos abonnés [La traque des axions, nouvelle coqueluche des physiciens](https://www.lemonde.fr/sciences/article/2025/06/09/la-traque-des-axions-nouvelle-coqueluche-des-physiciens_6611759_1650684.html)      Lire plus tard      Le Monde avec AFP \n\n     L’espace des contributions est réservé aux abonnés. Abonnez\\-vous pour accéder à cet espace d’échange et contribuer à la discussion.    [S’abonner](https://abo.lemonde.fr/?lmd_medium=BOUTONS_LMFR&lmd_campaign=CONTRIBUTION_ARTICLE)     Contribuer\n\n [Réutiliser ce contenu](https://www.lemonde.fr/syndication/ \"Réutiliser ce contenu\")                                Édition du jour\n\n Daté du jeudi 31 juillet\n\n <img src='data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 276 201'%3E%3C/svg%3E' alt='' title='' width='276' height='201' /> [Lire le journal numérique](https://journal.lemonde.fr/)          [<img src='' alt='' title='' width='' height='36' /> ![]() Culture générale Des leçons interactives par la rédaction pour tester vos connaissances.  Découvrir](https://www.lemonde.fr/memorable/quiz-et-questions-de-culture-generale)         [<img src='' alt='' title='' width='300' height='280' />](https://www.lemonde.fr/chaleur-humaine/article/2025/01/30/mesurez-votre-impact-environnement-avec-le-calculateur-d-empreinte-carbone-et-eau_6523433_6125299.html?lmd_medium=bizdev&lmd_campaign=services_partenaire_lmfr&lmd_creation=ademe)",
-    "score": 0.42882082,
-}
-result_5 = {
-    "url": "https://www.franceinfo.fr/economie/budget/",
-    "content": "Budget 2025 de la France \\- actualité et info en direct\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\ncontenu principal\n\n\n\n\n\n[<img src='https://www.franceinfo.fr/assets/components/headers/header/img/franceinfo-1d7b76a5.svg' alt='' title='' width='161' height='40' />\n\n<img src='data:image/svg+xml,%3Csvg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 161 40\"%3E%3C/svg%3E' alt='' title='' width='161' height='40' />\nAccueil France Info](/ \"Accueil France Info\")\n\n* [Rechercher une actualité\nRecherche](/recherche/)\n* [Direct TV\nTV](/en-direct/tv.html)\n* [Direct radio\nRadio](/en-direct/radio.html)\n* [Le live\nLive](/en-direct/)\n* Services\n\n\n\n\n\t+ [Newsletters\n\t (Nouvelle fenêtre)](https://www.francetelevisions.fr/abonnements/information)\n\t+ [Météo\n\t (Nouvelle fenêtre)](https://meteo.franceinfo.fr/)\n\t+ [Jeux\n\t (Nouvelle fenêtre)](https://jeux.franceinfo.fr)\n\t+ [Scolarité](/societe/education/scolarite/)\n* Mon\xa0espace\n\n\n\n\n\t+ [S'inscrire](javascript:void(0))\n\t+ [Se connecter](javascript:void(0))\n\t+ [À lire plus tard](/mon-compte/lire-plus-tard)\n\t+ [Mes commentaires](/mon-compte/interactions)\n\t+ [Mes newsletters](/mon-compte/newsletters)\n\t+ [Mes informations](javascript:void(0))\n\t+ [Se déconnecter](javascript:void(0))\n\n\n\n\n\n\n* [Accueil](/)\n* Menu\n* [Grands formats](/grands-formats/)\n* [Enquêtes](/enquetes-franceinfo/)\n* [Vrai ou faux](/vrai-ou-fake/)\n* [Droits de douane](/monde/usa/droits-de-douane/)\n* [Guerre dans la bande de Gaza](/monde/proche-orient/israel-palestine/)\n* [Loi Duplomb](/environnement/loi-duplomb/)\n\n\n\n\n\n\n\n<img src='https://www.franceinfo.fr/assets/components/headers/header/img/franceinfo-1d7b76a5.svg' alt='France Info' title='' width='129' height='32' />\n\n<img src='data:image/svg+xml,%3Csvg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 129 32\"%3E%3C/svg%3E' alt='France Info' title='' width='129' height='32' />\n\n\n\n\n\n\n\n\n\n\n\nRechercher sur franceinfo\n\n\n\n\n\n\nAnnuler la saisie\n\n\n\n\n* [Accueil](/)\n\n\n\n\n En ce moment\n \n\n* [Droits de douane](/monde/usa/droits-de-douane/)\n* [Guerre dans la bande de Gaza](/monde/proche-orient/israel-palestine/)\n* [Loi Duplomb](/environnement/loi-duplomb/)\n\n\n\n\n* [Grands formats](/grands-formats/)\n* [Enquêtes](/enquetes-franceinfo/)\n* [Vrai ou faux](/vrai-ou-fake/)\n\n\n\n\n Rubriques\n \n\n* [L'actu pour les jeunes](/l-actu-pour-les-jeunes/)\n* [Une info transparente](/une-information-transparente-franceinfo/)\n* Monde\n* Europe\n* Faits\\-divers\n* Politique\n* Société\n* Environnement\n* Sport\n* Culture\n* Éco / Conso\n* Santé\n* Sciences \\& technologies\n* Météo\nMétéo\n* Jeux\nJeux\n\n\n\n* Services\n\n\n\nRecevez l'essentiel de l'actualité et restez à jour avec nos newsletters\n\n\n[découvrir nos newsletters\n(Nouvelle fenêtre)](https://www.francetelevisions.fr/abonnements/information)\n\n\n\n Continuez votre exploration\n \n\n* [France 3 régions (nouvelle fenêtre)\nFrance 3 régions](https://france3-regions.franceinfo.fr/)\n* [Outremer la 1ère (nouvelle fenêtre)\nOutremer la 1ère](https://la1ere.franceinfo.fr/)\n* france TV\nfrance TV\n* radio france\nradio france\n\n\n\n\n\n\nServices\n\nServices\n* [Newsletters (nouvelle fenêtre)\nNewsletters](https://www.francetelevisions.fr/abonnements/information)\n* [Météo (nouvelle fenêtre)\nMétéo](https://meteo.franceinfo.fr/)\n* [Jeux (nouvelle fenêtre)\nJeux](https://jeux.franceinfo.fr)\n* [Scolarité](/societe/education/scolarite/)\n\n\n\n\n\n\nMonde\n\nMonde\n\n[voir toute l'actu Monde](/monde/)\n\n\n\n En ce moment\n \n\n* [Droits de douane](/monde/usa/droits-de-douane/)\n* [Guerre dans la bande de Gaza](/monde/proche-orient/israel-palestine/)\n* [Guerre en Ukraine](/monde/europe/manifestations-en-ukraine/)\n* [Affaire Jeffrey Epstein](/monde/usa/affaire-jeffrey-epstein/)\n* [Guerre au Proche\\-Orient](/monde/proche-orient/guerre/)\n* [Donald Trump \\- Président des États\\-Unis](/monde/usa/presidentielle/donald-trump/)\n* [Yémen](/monde/proche-orient/yemen/)\n* [Voir plus de sujets monde](/monde/)\n\n\n\n\n Rubriques",
-    "score": 0.6129482,
-}
-
-
-class MockedWebSearchManager:
-    """
-    A class to manage web search operations.
-
-    This is an abstract base class that should be implemented
-    for specific web search managers.
-    """
-
-    def web_search(self, query: str) -> RAGWebResults:
-        """
-        Fake a web search.
-
-        Args:
-            query (str): The search query.
-
-        Returns:
-            RAGWebResults: A Searches object containing the search results.
-        """
-        logger.info("Mocked web search called with query: %s", query)
-        return RAGWebResults(
-            data=[
-                RAGWebResult(**result_1),
-                RAGWebResult(**result_2),
-                RAGWebResult(**result_3),
-                RAGWebResult(**result_4),
-                RAGWebResult(**result_5),
-            ],
-            usage=RAGWebUsage(
-                prompt_tokens=0,  # Assuming no prompt tokens are provided in this mock
-                completion_tokens=0,  # Assuming no completion tokens are provided in this mock
-            ),  # Assuming no usage data is provided in this mock
-        )

src/backend/chat/agents/base.py

@@ -1,195 +0,0 @@
-"""Base module for PydanticAI agents."""
-
-import dataclasses
-import logging
-
-from django.conf import settings
-from django.core.exceptions import ImproperlyConfigured
-
-import httpx
-from pydantic_ai import Agent
-from pydantic_ai.models import get_user_agent
-from pydantic_ai.profiles import ModelProfile
-from pydantic_ai.toolsets import FunctionToolset
-
-from chat.tools import get_pydantic_tools_by_name
-
-logger = logging.getLogger(__name__)
-
-
-def prepare_custom_model(configuration: "chat.llm_configuration.LLModel"):
-    """
-    Prepare a custom model instance based on the provided configuration.
-
-    Only few providers are supported at the moment, according to our needs.
-    We define custom models/providers to be able to keep specific configuration
-    when needed.
-    """
-    # pylint: disable=import-outside-toplevel
-
-    match configuration.provider.kind:
-        case "mistral":
-            import pydantic_ai.models.mistral as mistral_models  # noqa: PLC0415
-            from mistralai import TextChunk as MistralTextChunk  # noqa: PLC0415
-            from mistralai import ThinkChunk as MistralThinkChunk  # noqa: PLC0415
-            from mistralai.types.basemodel import Unset as MistralUnset  # noqa: PLC0415
-            from pydantic_ai.providers.mistral import MistralProvider  # noqa: PLC0415
-
-            # --- Monkey patch for pydantic_ai.models.mistral._map_content ---
-            # pylint: disable=protected-access
-
-            # ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠
-            # |  This workaround is fragile and only works because we are in streaming mode.  |
-            # ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠ WARNING ⚠
-
-            # The original _map_content raises exceptions for some when responses
-            # contains citation/reference data, which is the case anytime we use
-            # web search or other RAG tool (https://docs.mistral.ai/capabilities/citations/).
-            # We make the patch idempotent using a sentinel attribute so repeated calls
-            # to prepare_custom_model do not re-wrap and do not cause recursive calls.
-            if not getattr(mistral_models, "__safe_map_patched__", False):
-                _original_map_content = mistral_models._map_content  # noqa: SLF001
-
-                def _safe_map_content(content):
-                    """
-                    A safe version of _map_content that ignores unsupported data types.
-
-                    WARNING: this is a monkey patch and may break if the original
-                    function changes in future versions of pydantic_ai.
-                    Current version: pydantic_ai v1.0.18
-                    """
-                    text: str | None = None
-                    thinking: list[str] = []
-
-                    if isinstance(content, MistralUnset) or not content:
-                        return None, []
-
-                    if isinstance(content, list):
-                        for chunk in content:
-                            if isinstance(chunk, MistralTextChunk):
-                                text = (text or "") + chunk.text
-                            elif isinstance(chunk, MistralThinkChunk):
-                                for thought in chunk.thinking:
-                                    if thought.type == "text":  # pragma: no branch
-                                        thinking.append(thought.text)
-                            else:
-                                logger.info(  # pragma: no cover
-                                    "Other data types like (Image, Reference) are not yet "
-                                    "supported,  got %s",
-                                    type(chunk),
-                                )
-                    elif isinstance(content, str):
-                        text = content
-
-                    # Note: Check len to handle potential mismatch between function calls and
-                    # responses from the API.
-                    # (`msg: not the same number of function class and responses`)
-                    if text == "":  # pragma: no cover
-                        text = None
-
-                    return text, thinking
-
-                # Replace the original module-level function
-                mistral_models._map_content = _safe_map_content  # noqa: SLF001
-                mistral_models.__safe_map_patched__ = True
-            # pylint: enable=protected-access
-            # --- End monkey patch ---
-
-            return mistral_models.MistralModel(
-                model_name=configuration.model_name,
-                profile=(
-                    ModelProfile(**configuration.profile.dict(exclude_unset=True))
-                    if configuration.profile
-                    else None
-                ),
-                provider=MistralProvider(
-                    api_key=configuration.provider.api_key,
-                    base_url=configuration.provider.base_url,
-                    # Disable the use of cached client
-                    http_client=httpx.AsyncClient(
-                        timeout=httpx.Timeout(timeout=600, connect=5),
-                        headers={"User-Agent": get_user_agent()},
-                    ),
-                ),
-            )
-        case "openai":
-            from pydantic_ai.models.openai import OpenAIChatModel  # noqa: PLC0415
-            from pydantic_ai.profiles.openai import OpenAIModelProfile  # noqa: PLC0415
-            from pydantic_ai.providers.openai import OpenAIProvider  # noqa: PLC0415
-
-            if configuration.profile and (
-                _config_profile := configuration.profile.dict(exclude_unset=True)
-            ):
-                # set some defaults if not provided, see openai_model_profile which
-                # defines them for known models
-                _model_profile_params = {
-                    "supports_json_schema_output": True,
-                    "supports_json_object_output": True,
-                }
-                _model_profile_params.update(_config_profile)
-                profile = OpenAIModelProfile(**_model_profile_params)
-            else:
-                profile = None
-
-            return OpenAIChatModel(
-                model_name=configuration.model_name,
-                profile=profile,
-                provider=OpenAIProvider(
-                    base_url=configuration.provider.base_url,
-                    api_key=configuration.provider.api_key,
-                ),
-            )
-        case _:
-            raise ImproperlyConfigured(
-                f"Unsupported provider kind '{configuration.provider.kind}' for custom model."
-            )
-
-
-@dataclasses.dataclass(init=False)
-class BaseAgent(Agent):
-    """
-    Base class for PydanticAI agents.
-
-    This class initializes the agent with model from configuration.
-    """
-
-    def __init__(self, *, model_hrid, **kwargs):
-        """Initialize the agent with model configuration from settings."""
-        _ignored_kwargs = {"model", "system_prompt", "tools", "toolsets"}
-        if set(kwargs).intersection(_ignored_kwargs):
-            raise ValueError(f"{_ignored_kwargs} arguments must not be provided.")
-
-        try:
-            self.configuration = settings.LLM_CONFIGURATIONS[model_hrid]
-        except KeyError as exc:
-            raise ImproperlyConfigured(
-                f"LLM model configuration '{model_hrid}' not found."
-            ) from exc
-
-        if self.configuration.is_custom:
-            _model_instance = prepare_custom_model(self.configuration)
-        else:
-            # In this case, we rely on PydanticAI's built-in model registry
-            # and configuration: check pydantic_ai.models.KnownModelName
-            # and pydantic_ai.models.infer_model()
-            _model_instance = self.configuration.model_name
-
-        _system_prompt = self.configuration.system_prompt
-        _base_toolset = (
-            [
-                FunctionToolset(
-                    tools=[
-                        get_pydantic_tools_by_name(tool_name)
-                        for tool_name in self.configuration.tools
-                    ]
-                )
-            ]
-            if self.configuration.tools
-            else None
-        )
-
-        _tools = [get_pydantic_tools_by_name(tool_name) for tool_name in self.configuration.tools]
-
-        super().__init__(
-            model=_model_instance, system_prompt=_system_prompt, tools=_tools, **kwargs
-        )

src/backend/chat/agents/conversation.py

@@ -1,52 +0,0 @@
-"""Build the main conversation agent."""
-
-import dataclasses
-import logging
-
-from django.utils import formats, timezone
-
-from core.enums import get_language_name
-
-from .base import BaseAgent
-
-logger = logging.getLogger(__name__)
-
-
-@dataclasses.dataclass(init=False)
-class ConversationAgent(BaseAgent):
-    """Conversation agent with custom behavior."""
-
-    def __init__(self, *, language=None, **kwargs):
-        """Initialize the conversation agent."""
-        super().__init__(**kwargs)
-
-        @self.system_prompt
-        def add_the_date() -> str:
-            """
-            Dynamic system prompt function to add the current date.
-
-            Warning: this will always use the date in the server timezone,
-            not the user's timezone...
-            """
-            _formatted_date = formats.date_format(timezone.now(), "l d/m/Y", use_l10n=False)
-            return f"Today is {_formatted_date}."
-
-        @self.system_prompt
-        def enforce_response_language() -> str:
-            """Dynamic system prompt function to set the expected language to use."""
-            return f"Answer in {get_language_name(language).lower()}." if language else ""
-
-    def get_web_search_tool_name(self) -> str | None:
-        """
-        Get the name of the web search tool if available.
-
-        If several are available, return the first one found.
-
-        Warning, this says the tool is available, not that
-        it (the tool/feature) is enabled for the current conversation.
-        """
-        for toolset in self.toolsets:
-            for tool in toolset.tools.values():
-                if tool.name.startswith("web_search_"):
-                    return tool.name
-        return None

src/backend/chat/agents/local_media_url_processors.py

@@ -1,94 +0,0 @@
-"""
-ImageUrl processors and utilities.
-
-Allow to manage local image URLs in messages, replacing them with presigned S3 URLs
-for the LLM to access them, and then reverting them back to local URLs when
-storing the messages in the database.
-"""
-
-from typing import Dict, Iterable
-
-from pydantic_ai import DocumentUrl, ImageUrl, ModelMessage, ModelRequest, UserPromptPart
-
-from core.file_upload.utils import generate_retrieve_policy
-
-from chat.models import ChatConversation
-
-
-def update_local_urls(
-    conversation: ChatConversation,
-    contents: Iterable[ImageUrl | DocumentUrl],
-    updated_url: Dict[str, str] | None = None,
-) -> Iterable[ImageUrl | DocumentUrl]:
-    """
-    Replace local image or document URLs in the content list to use presigned S3 URLs.
-    ⚠️Be careful, `media_contents` are replaced in place.
-
-    Args:
-        conversation (ChatConversation): The chat conversation object.
-        contents (Iterable[ImageUrl | DocumentUrl]): Iterable of UserContent objects.
-        updated_url (Dict[str, str], optional): Dictionary to store
-            mapping of original URLs to updated URLs.
-    Returns:
-        Iterable[ImageUrl | DocumentUrl]: Updated iterable of UserContent objects
-            with presigned URLs.
-    """
-    # When images are stored locally, there is no host in the URL, so we can
-    # just check if the URL starts, frontend adds a prefix `/media-key/` to the key.
-    local_media_url_prefix = "/media-key/"
-    local_media_url_prefix_len = len(local_media_url_prefix)
-
-    # Filter only ImageUrl contents
-    media_contents = (c for c in contents if isinstance(c, (ImageUrl, DocumentUrl)))
-
-    # Replace URLs with presigned URLs
-    for content in media_contents:
-        idx = content.url.find(local_media_url_prefix)
-
-        if idx == 0:
-            _initial_url = str(content.url)
-            key = content.url[local_media_url_prefix_len:]
-
-            # Security check: ensure the image belongs to the conversation, if yes,
-            # the user had access to the endpoint, so they have access to the image.
-            if not key.startswith(f"{conversation.pk}/"):
-                # The LLM will throw an error when trying to access the image,
-                # this is not perfect, but this should never happen in practice,
-                # except if the user tampers with the conversation.
-                continue
-
-            content.url = generate_retrieve_policy(key)
-            if updated_url is not None:
-                updated_url[content.url] = _initial_url
-
-    return contents
-
-
-def update_history_local_urls(
-    conversation: ChatConversation, messages: list[ModelMessage]
-) -> list[ModelMessage]:
-    """
-    Replace local image/documents URLs in the message list to use presigned S3 URLs.
-
-    ⚠️Be careful, `messages` are replaced in place.
-
-    We don't need to store the mapping of updated URLs to original URLs here because
-    this function is used when sending the history to the LLM (which is already stored
-    in the database with local URLs).
-
-    Args:
-        messages (list[ModelMessage]): List of ModelMessage objects.
-    Returns:
-        list[ModelMessage]: Updated list of ModelMessage objects with presigned URLs.
-    """
-    # Filter only ModelRequest messages
-    requests = (msg for msg in messages if isinstance(msg, ModelRequest))
-
-    for message in requests:
-        # Filter only UserPromptPart parts
-        user_parts = (part for part in message.parts if isinstance(part, UserPromptPart))
-
-        for part in user_parts:
-            update_local_urls(conversation, part.content)
-
-    return messages

src/backend/chat/agents/summarize.py

@@ -1,85 +0,0 @@
-"""Build the summarization agent."""
-
-import dataclasses
-import logging
-
-from django.conf import settings
-from django.core.files.storage import default_storage
-
-from asgiref.sync import sync_to_async
-from pydantic_ai import RunContext
-from pydantic_ai.messages import ToolReturn
-
-from .base import BaseAgent
-
-logger = logging.getLogger(__name__)
-
-
-@dataclasses.dataclass(init=False)
-class SummarizationAgent(BaseAgent):
-    """Create a Pydantic AI summarization Agent instance with the configured settings"""
-
-    def __init__(self, **kwargs):
-        """Initialize the agent with the configured model."""
-        super().__init__(
-            model_hrid=settings.LLM_SUMMARIZATION_MODEL_HRID,
-            output_type=str,
-            **kwargs,
-        )
-
-
-@sync_to_async
-def read_document_content(doc):
-    """Read document content asynchronously."""
-    with default_storage.open(doc.key) as f:
-        return doc.file_name, f.read().decode("utf-8")
-
-
-async def hand_off_to_summarization_agent(ctx: RunContext) -> ToolReturn:
-    """
-    Generate a complete, ready-to-use summary of the documents in context
-    (do not request the documents to the user).
-    Return this summary directly to the user WITHOUT any modification,
-    or additional summarization.
-    The summary is already optimized and MUST be presented as-is in the final response
-    or translated preserving the information.
-    """
-    summarization_agent = SummarizationAgent()
-
-    prompt = (
-        "Do not mention the user request in your answer.\n"
-        "User request:\n"
-        "{user_prompt}\n\n"
-        "Document contents:\n"
-        "{documents_prompt}\n"
-    )
-    text_attachment = await sync_to_async(list)(
-        ctx.deps.conversation.attachments.filter(
-            content_type__startswith="text/",
-        )
-    )
-
-    documents = [await read_document_content(doc) for doc in text_attachment]
-
-    documents_prompt = "\n\n".join(
-        [
-            (f"<document>\n<name>\n{name}\n</name>\n<content>\n{content}\n</content>\n</document>")
-            for name, content in documents
-        ]
-    )
-
-    formatted_prompt = prompt.format(
-        user_prompt=ctx.prompt,
-        documents_prompt=documents_prompt,
-    )
-
-    logger.debug("Summarize prompt: %s", formatted_prompt)
-
-    response = await summarization_agent.run(formatted_prompt, usage=ctx.usage)
-
-    logger.debug("Summarize response: %s", response)
-
-    return ToolReturn(
-        return_value=response.output,
-        metadata={"sources": {doc[0] for doc in documents}},
-    )

src/backend/chat/ai_sdk_types.py

@@ -22,7 +22,7 @@ class ToolCall(BaseModel):
 
     toolCallId: str
     toolName: str
-    args: Optional[Dict[str, Any]] = None
+    args: Dict[str, Any]
 
 
 class ToolResult(BaseModel):
@@ -38,7 +38,7 @@ class ToolResult(BaseModel):
 
     toolCallId: str
     toolName: str
-    args: Optional[Dict[str, Any]] = None
+    args: Dict[str, Any]
     result: Any
 
 
@@ -177,21 +177,15 @@ class ToolInvocationUIPart(BaseModel):
 
 class LanguageModelV1Source(BaseModel):
     """
-    Represents a source that has been used as input to generate the response.
+    Represents source information from a language model.
 
     Attributes:
-        sourceType: A URL source. This is return by web search RAG models.
-        id: The ID of the source.
-        url: The URL of the source.
-        title: The title of the source.
-        providerMetadata: Additional provider metadata for the source.
+        source_type: The type of source.
+        details: Additional details about the source.
     """
 
-    sourceType: Literal["url"]
-    id: str
-    url: str
-    title: Optional[str] = None
-    providerMetadata: Dict[str, Any]  # LanguageModelV1ProviderMetadata
+    source_type: str
+    details: Dict[str, Any]
 
 
 class SourceUIPart(BaseModel):

src/backend/chat/clients/async_to_sync.py

@@ -1,49 +0,0 @@
-"""
-Helpers to manage async objects in a synchronous context.
-
-This is not optimal, but we would prefer to stay in a synchronous context
-for now.
-"""
-
-import asyncio
-import queue
-import threading
-
-from chat.clients.exceptions import StreamCancelException
-
-
-def convert_async_generator_to_sync(async_gen):
-    """Convert an async generator to a sync generator."""
-    q = queue.Queue()
-    sentinel = object()
-    exc_sentinel = object()
-
-    async def run_async_gen():
-        try:
-            async for async_item in async_gen:
-                q.put(async_item)
-        except StreamCancelException:
-            # Handle cancellation gracefully, do not put anything in the queue
-            q.put(sentinel)
-        except Exception as exc:  # pylint: disable=broad-except #noqa: BLE001
-            q.put((exc_sentinel, exc))
-        finally:
-            q.put(sentinel)
-
-    def start_async_loop():
-        asyncio.run(run_async_gen())
-
-    thread = threading.Thread(target=start_async_loop, daemon=True)
-    thread.start()
-
-    try:
-        while True:
-            item = q.get()
-            if item is sentinel:
-                break
-            if isinstance(item, tuple) and item[0] is exc_sentinel:
-                # re-raise the exception in the sync context
-                raise item[1]
-            yield item
-    finally:
-        thread.join()

src/backend/chat/clients/exceptions.py

@@ -1,17 +0,0 @@
-"""Module containing custom exceptions for chat clients."""
-
-
-class WebSearchEmptyException(Exception):
-    """Exception raised when a web search returns no results."""
-
-    def __init__(self, message="Web search returned no results."):
-        self.message = message
-        super().__init__(self.message)
-
-
-class StreamCancelException(Exception):
-    """Exception raised when a streaming operation is cancelled."""
-
-    def __init__(self, message="Streaming operation was cancelled."):
-        self.message = message
-        super().__init__(self.message)

src/backend/chat/clients/langchain.py

@@ -0,0 +1,322 @@
+"""LangChainAgentService class for handling AI agent interactions using LangChain."""
+
+import asyncio
+import json
+import logging
+import queue
+import threading
+import uuid
+from contextlib import AsyncExitStack
+from typing import List
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+
+from langchain_core.messages import AIMessageChunk, ToolMessage
+from langchain_core.tools import tool
+
+from chat.ai_sdk_types import (
+    TextUIPart,
+    ToolInvocationPartialCall,
+    ToolInvocationResult,
+    ToolInvocationUIPart,
+    UIMessage,
+)
+from chat.mcp_servers import get_mcp_servers
+
+logger = logging.getLogger(__name__)
+
+# LangChain imports
+from langchain.agents import AgentType, create_react_agent, initialize_agent
+from langchain.chat_models import init_chat_model
+from langchain.schema import (
+    AIMessage,
+    BaseMessage,
+    FunctionMessage,
+    HumanMessage,
+    SystemMessage,
+)
+from langchain.tools import Tool
+
+
+@tool(parse_docstring=True)
+def get_current_weather(location: str, unit: str):
+    """
+    Get the current weather in a given location.
+
+    Args:
+        location (str): The city and state, e.g. San Francisco, CA.
+        unit (str): The unit of temperature, either 'celsius' or 'fahrenheit'.
+    """
+    return {
+        "location": location,
+        "temperature": 22 if unit == "celsius" else 72,
+        "unit": unit,
+    }
+
+
+def convert_async_generator_to_sync(async_gen):
+    """Convert an async generator to a sync generator."""
+    q = queue.Queue()
+    sentinel = object()
+
+    async def run_async_gen():
+        try:
+            async for item in async_gen:
+                q.put(item)
+        finally:
+            q.put(sentinel)
+
+    def start_async_loop():
+        asyncio.run(run_async_gen())
+
+    thread = threading.Thread(target=start_async_loop)
+    thread.start()
+
+    while True:
+        item = q.get()
+        if item is sentinel:
+            break
+        yield item
+
+    thread.join()
+
+
+class AIAgentService:
+    """Service class for AI-related operations using LangChain."""
+
+    def __init__(self, conversation):
+        """Ensure that the AI configuration is set properly."""
+        if settings.AI_API_KEY is None or settings.AI_MODEL is None:
+            raise ImproperlyConfigured("LangChainAgentService configuration not set")
+
+        self.model = init_chat_model(
+            openai_api_key=settings.AI_API_KEY,
+            model=settings.AI_MODEL,
+            model_provider="openai",
+            base_url=settings.AI_BASE_URL,
+        )
+        self.conversation = conversation
+
+    @staticmethod
+    def _convert_to_langchain_messages(messages: List[UIMessage]) -> List[BaseMessage]:
+        lc_messages = []
+        for message in messages:
+            logger.info(f"Converting message: {message}")
+            content = []
+            # Handle main parts
+            for part in message.parts:
+                if part.type == "text":
+                    content.append({"type": "text", "text": part.text})
+                    # content.append(part.text)
+                elif part.type == "tool-invocation":
+                    # Represent tool invocation as a FunctionMessage
+                    tool_invocation = part.toolInvocation
+                    lc_messages.append(
+                        FunctionMessage(
+                            name=tool_invocation.toolName, content=json.dumps(tool_invocation.args)
+                        )
+                    )
+                elif part.type == "image":
+                    # Represent image as a string with a tag or metadata
+                    content.append(
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": part.url},
+                        }
+                    )
+                    # content.append(f"[image: {getattr(part, 'url', '')}]")
+                elif part.type == "file":
+                    # Represent file as a string with a tag or metadata
+                    content.append(
+                        f"[file: {getattr(part, 'name', '')} {getattr(part, 'url', '')}]"
+                    )
+                # Add more types as needed
+
+            # Handle experimental_attachments if present
+            if hasattr(message, "experimental_attachments") and message.experimental_attachments:
+                for attachment in message.experimental_attachments:
+                    if getattr(attachment, "contentType", "").startswith("image"):
+                        content.append(
+                            {
+                                "type": "image_url",
+                                "image_url": {"url": attachment.url},
+                            }
+                        )
+                    elif getattr(attachment, "contentType", "").startswith("text"):
+                        content.append(
+                            f"[file: {getattr(attachment, 'name', '')} {getattr(attachment, 'url', '')}]"
+                        )
+
+            # Compose the message
+            lc_messages.append(
+                {
+                    "role": message.role,
+                    "content": content,
+                }
+            )
+            # if message.role == "system":
+            #    lc_messages.append(SystemMessage(content=content))
+            # elif message.role == "user":
+            #    lc_messages.append(HumanMessage(content=content))
+            # elif message.role == "assistant":
+            #    lc_messages.append(AIMessage(content=content))
+            # FunctionMessage already appended above for tool-invocation
+            # Add more roles/types as needed
+        return lc_messages
+
+    def stream_data(self, messages: List[UIMessage]):  # noqa: PLR0912
+        lc_messages = self._convert_to_langchain_messages(messages)
+        logger.info("[LangChain stream_data_async] Received messages: %s", lc_messages)
+
+        tools = [get_current_weather]
+
+        from langchain_core.prompts import ChatPromptTemplate
+        from langgraph.prebuilt import create_react_agent
+        from langgraph.prebuilt.chat_agent_executor import AgentState
+
+        prompt = ChatPromptTemplate.from_messages(
+            [
+                ("system", "You are a helpful assistant."),
+                ("placeholder", "{messages}"),
+            ]
+        )
+
+        langgraph_agent_executor = create_react_agent(self.model, tools, prompt=prompt)
+
+        finish_reason = "stop"
+
+        _current_tool_call_id = None
+        _current_tool_call_arguments = ""
+        for stream_mode, step in langgraph_agent_executor.stream(
+            {"messages": lc_messages},
+            stream_mode=["messages", "values"],
+        ):
+            # (
+            #     AIMessageChunk(
+            #         content=" with",
+            #         additional_kwargs={},
+            #         response_metadata={},
+            #         id="run--c9fbf725-0ca0-408f-8158-da347a26b64b",
+            #     ),
+            #     {
+            #         "langgraph_step": 1,
+            #         "langgraph_node": "agent",
+            #         "langgraph_triggers": ("branch:to:agent",),
+            #         "langgraph_path": ("__pregel_pull", "agent"),
+            #         "langgraph_checkpoint_ns": "agent:980f9c50-3cfa-6522-f9be-de139f9b4ece",
+            #         "checkpoint_ns": "agent:980f9c50-3cfa-6522-f9be-de139f9b4ece",
+            #         "ls_provider": "openai",
+            #         "ls_model_name": "ai/smollm2",
+            #         "ls_model_type": "chat",
+            #         "ls_temperature": None,
+            #     },
+            # )
+            if stream_mode == "messages":
+                chunk, metadata = step
+
+                try:
+                    logger.info(
+                        "[LangChain stream_data_async] Received chunk: %s %s", type(chunk), chunk
+                    )
+                    if isinstance(chunk, AIMessageChunk):
+                        if chunk.tool_call_chunks:
+                            for tool_call_chunk in chunk.tool_call_chunks:
+                                # Handle tool call chunks
+                                if tool_call_chunk["id"]:
+                                    _current_tool_call_id = tool_call_chunk["id"]
+                                    _tool_name = tool_call_chunk["name"]
+                                    logger.info(
+                                        "[LangChain stream_data_async] Tool call chunk: %s %s",
+                                        _current_tool_call_id,
+                                        _tool_name,
+                                    )
+                                    # yield f'b:{{"toolCallId":"{_current_tool_call_id}","toolName":"{_tool_name}"}}\n'
+
+                                else:
+                                    _argument_delta = tool_call_chunk["args"]
+                                    _current_tool_call_arguments += _argument_delta
+                                    logger.info(
+                                        "[LangChain stream_data_async] Tool call argument delta: %s %s",
+                                        _current_tool_call_id,
+                                        _argument_delta,
+                                    )
+                                    # yield f'c:{{"toolCallId":"{_current_tool_call_id}","argsTextDelta":"{_argument_delta}"}}\n'
+                        elif chunk.response_metadata.get("finish_reason"):
+                            finish_reason = chunk.response_metadata["finish_reason"]
+                            logger.info(
+                                "[LangChain stream_data_async] Finish reason: %s", finish_reason
+                            )
+                        else:
+                            yield f"0:{json.dumps(chunk.content)}\n"
+                    elif isinstance(chunk, ToolMessage) and chunk.content:
+                        _tool_call = {
+                            "toolCallId": chunk.tool_call_id,
+                            "toolName": chunk.name,
+                            "args": json.loads(_current_tool_call_arguments),
+                        }
+                        yield f"9:{json.dumps(_tool_call)}\n"
+                        # content='{"location": "Paris, France", "temperature": 22, "unit": "celsius"}' name='get_current_weather' id='159415d7-046d-43bf-982e-8a69cb50a486' tool_call_id='qk6PXRocvUzh9QTQS6HQNI5qpNujsrzr'
+                        _tool_result_part = {
+                            "toolCallId": chunk.tool_call_id,
+                            "toolName": chunk.name,
+                            "result": json.loads(chunk.content),
+                        }
+                        _current_tool_call_id = None
+                        _current_tool_call_arguments = ""
+                        yield f"a:{json.dumps(_tool_result_part)}\n"
+                    else:
+                        yield f"0:{json.dumps(chunk.content)}\n"
+
+                except Exception as e:
+                    logger.exception("Error in LangChain stream_data_async")
+                    yield f"3:{json.dumps(str(e))}\n"
+                    finish_reason = "error"
+
+            elif stream_mode == "values":
+                logger.info("[LangChain stream_data_async] Received values: %s", step)
+                last_message = step["messages"][-1]
+                # if isinstance(last_message, AIMessage) and last_message.tool_calls:
+                #    for tool_call in last_message.tool_calls:
+                #        _tool_call = {
+                #            "toolCallId": tool_call["id"],
+                #            "toolName": tool_call["name"],
+                #            "args": tool_call["args"],
+                #        }
+                #        yield f"9:{json.dumps(_tool_call)}\n"
+
+        # Simulate finish message
+        _finish_message = {
+            "finishReason": finish_reason,
+            "usage": {},  # LangChain does not provide token usage by default
+        }
+        yield f"d:{json.dumps(_finish_message)}\n"
+
+    def _update_conversation(
+        self,
+        input_messages,
+        result_raw_responses,
+        response_usage,
+    ):
+        # For now, just save the input and output messages
+        # self.conversation.langchain_messages = input_messages
+        self.conversation.messages = self.conversation.ui_messages + [
+            {"content": str(result_raw_responses)}
+        ]
+        self.conversation.save()
+
+    def _convert_langchain_output_to_ui_messages(self, output):
+        # Convert LangChain output to UI messages (simple version)
+        ui_messages = []
+        if isinstance(output, str):
+            text_parts = [TextUIPart(type="text", text=output)]
+            ui_messages.append(
+                UIMessage(
+                    id=str(uuid.uuid4()),
+                    role="assistant",
+                    parts=text_parts,
+                    content=output,
+                )
+            )
+        # Add more conversion logic as needed
+        return ui_messages

src/backend/chat/clients/openai.py

@@ -0,0 +1,408 @@
+"""AIAgentService class for handling AI agent interactions."""
+
+import asyncio
+import json
+import logging
+import queue
+import threading
+import uuid
+from contextlib import AsyncExitStack
+from typing import List
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+
+from agents import Agent, ModelResponse, OpenAIChatCompletionsModel, Runner, Usage
+from asgiref.sync import sync_to_async
+from openai import AsyncOpenAI
+from openai.types.responses import ResponseInputItemParam, ResponseOutputItem
+from openai.types.responses.response_usage import (
+    InputTokensDetails,
+    OutputTokensDetails,
+    ResponseUsage,
+)
+
+from chat.ai_sdk_types import (
+    TextUIPart,
+    ToolInvocationPartialCall,
+    ToolInvocationResult,
+    ToolInvocationUIPart,
+    UIMessage,
+)
+from chat.mcp_servers import get_mcp_servers
+
+logger = logging.getLogger(__name__)
+
+
+def convert_async_generator_to_sync(async_gen):
+    """Convert an async generator to a sync generator."""
+    q = queue.Queue()
+    sentinel = object()
+
+    async def run_async_gen():
+        try:
+            async for item in async_gen:
+                q.put(item)
+        finally:
+            q.put(sentinel)
+
+    def start_async_loop():
+        asyncio.run(run_async_gen())
+
+    thread = threading.Thread(target=start_async_loop)
+    thread.start()
+
+    while True:
+        item = q.get()
+        if item is sentinel:
+            break
+        yield item
+
+    thread.join()
+
+
+class AIAgentService:
+    """Service class for AI-related operations."""
+
+    def __init__(self, conversation):
+        """Ensure that the AI configuration is set properly."""
+        if settings.AI_BASE_URL is None or settings.AI_API_KEY is None or settings.AI_MODEL is None:
+            raise ImproperlyConfigured("AIChatService configuration not set")
+
+        self.model = OpenAIChatCompletionsModel(
+            model=settings.AI_MODEL,
+            openai_client=AsyncOpenAI(
+                base_url=settings.AI_BASE_URL,
+                api_key=settings.AI_API_KEY,
+            ),
+        )
+        self.conversation = conversation
+
+    @staticmethod
+    def _convert_to_openai_messages(
+        messages: List[UIMessage],
+    ) -> List[ResponseInputItemParam]:
+        """Convert UI messages to OpenAI format."""
+        openai_messages = []
+
+        for message in messages:
+            content_parts = []
+            tool_calls = []
+
+            for part in message.parts:
+                match part.type:
+                    case "text":
+                        content_parts.append(
+                            {
+                                "type": "input_text" if message.role == "user" else "output_text",
+                                "text": part.text,
+                            }
+                        )
+
+                    case "image":
+                        content_parts.append(
+                            {
+                                "type": "input_image",
+                                "image_url": part.url,
+                            }
+                        )
+
+                    case "file":
+                        content_parts.append(
+                            {
+                                "type": "input_file",
+                                "file_data": part.url,
+                                "filename": part.name,
+                            }
+                        )
+
+                    case "tool-invocation":
+                        # Extract the tool invocation data
+                        tool_invocation = part.toolInvocation
+                        tool_calls.append(
+                            {
+                                "call_id": tool_invocation.toolCallId,
+                                "type": "function_call",
+                                "name": tool_invocation.toolName,
+                                "arguments": json.dumps(tool_invocation.args),
+                                "status": tool_invocation.state,
+                            }
+                        )
+
+                    case _:
+                        logger.warning("Unrecognized part type: %s in part: %s", part.type, part)
+
+            # Add experimental attachments if they exist
+            if hasattr(message, "experimental_attachments") and message.experimental_attachments:
+                for attachment in message.experimental_attachments:
+                    if attachment.contentType.startswith("image"):
+                        content_parts.append(
+                            {
+                                "type": "input_image",
+                                "image_url": attachment.url,
+                            }
+                        )
+                    elif attachment.contentType.startswith("text"):
+                        content_parts.append(
+                            {
+                                "type": "input_file",
+                                "file_data": attachment.url,
+                                "filename": attachment.name,
+                            }
+                        )
+
+            # Add message with content parts if there are any
+            if content_parts:
+                openai_messages.append(
+                    {"role": message.role, "content": content_parts, "type": "message"}
+                )
+
+            # Add tool calls separately, will be merged by `items_to_messages`
+            openai_messages += tool_calls
+
+        return openai_messages
+
+    def stream_text(self, messages: List[UIMessage]):
+        """Simple generator to convert async generator to sync generator."""
+        async_generator = self.stream_text_async(messages)
+        return convert_async_generator_to_sync(async_generator)
+
+    async def stream_text_async(self, messages: List[UIMessage]):
+        """Async generator for streaming agent events."""
+        openai_messages = self._convert_to_openai_messages(messages)
+        logger.debug("[stream_data_async] Received messages: %s", openai_messages)
+
+        async with AsyncExitStack() as stack:
+            initialized_mcp_servers = [
+                await stack.enter_async_context(mcp_server) for mcp_server in get_mcp_servers()
+            ]
+            agent = Agent(
+                name=settings.AI_AGENT_NAME,
+                instructions=settings.settings.AI_AGENT_INSTRUCTIONS,
+                model=self.model,
+                #tools=[agent_get_current_weather],
+                mcp_servers=initialized_mcp_servers,
+            )
+
+            result = Runner.run_streamed(
+                agent,
+                input=openai_messages,
+            )
+
+            async for event in result.stream_events():
+                logger.debug("[stream_text_async] Received event: %s", event)
+                if event.type == "raw_response_event":
+                    data = event.data
+                    logger.debug("[stream_text_async]  - data: %s", data)
+                    if data.type == "response.output_text.delta":
+                        yield data.delta
+
+            # At the end, save the response and yield the finish message part
+            _response_usage = Usage()
+            for raw_response in result.raw_responses:
+                _response_usage.add(raw_response.usage)
+
+            await sync_to_async(self._update_conversation)(
+                openai_messages, result.raw_responses, _response_usage
+            )
+
+    def stream_data(self, messages: List[UIMessage]):
+        """Simple generator to convert async generator to sync generator."""
+        async_generator = self.stream_data_async(messages)
+        return convert_async_generator_to_sync(async_generator)
+
+    async def stream_data_async(self, messages: List[UIMessage]):
+        """Async generator for streaming agent events."""
+        finish_reason = "stop"
+
+        openai_messages = self._convert_to_openai_messages(messages)
+        logger.debug("[stream_data_async] Received messages: %s", openai_messages)
+
+        async with AsyncExitStack() as stack:
+            initialized_mcp_servers = [
+                await stack.enter_async_context(mcp_server) for mcp_server in get_mcp_servers()
+            ]
+
+            # websearch_agent = Agent(
+            #     name="web search",
+            #     instructions=(
+            #         "You are a web search agent. "
+            #         "Your task is to search the web for up-to-date information."
+            #         " You will be called by the main agent to perform web searches."
+            #         " You will receive a query and return the search results."
+            #         " The results should be in the format of a list of dictionaries, "
+            #         "each containing 'link', 'title', and 'snippet' keys."
+            #         " If you cannot find any results, return an empty list."
+            #         " Do not include any other information in your response."
+            #         " Do not include any additional text or explanations."
+            #         " You must always annotate the response mentioning the url, title, and snippet."
+            #     ),
+            #     handoff_description="You are a web search agent. Your task is to search the web for up-to-date information.",
+            #     model=self.model,
+            #     tools=[agent_web_search_tavily],
+            # )
+
+            agent = Agent(
+                name=settings.AI_AGENT_NAME,
+                instructions=settings.AI_AGENT_INSTRUCTIONS,
+                model=self.model,
+                mcp_servers=initialized_mcp_servers,
+                # handoffs=[websearch_agent],
+            )
+            result = Runner.run_streamed(
+                agent,
+                input=openai_messages,
+            )
+
+            try:
+                async for event in result.stream_events():
+                    logger.debug("[stream_data_async] Received event: %s", event)
+
+                    if event.type == "raw_response_event":
+                        data = event.data
+                        if data.type == "response.output_text.delta":
+                            yield f"0:{json.dumps(data.delta)}\n"
+
+                        if hasattr(data, "finish_reason") and data.finish_reason:
+                            finish_reason = data.finish_reason
+
+                    elif event.type == "run_item_stream_event":
+                        item = event.item
+                        if item.type == "tool_call_item":
+                            _tool_call = {
+                                "toolCallId": item.raw_item.call_id,
+                                "toolName": item.raw_item.name,
+                                "args": (
+                                    json.loads(item.raw_item.arguments)
+                                    if hasattr(item.raw_item, "arguments")
+                                    else {}
+                                ),
+                            }
+                            yield f"9:{json.dumps(_tool_call)}\n"
+                        elif item.type == "tool_call_output_item":
+                            _tool_call_result = {
+                                "toolCallId": str(item.raw_item["call_id"]),
+                                "result": item.raw_item["output"],
+                            }
+                            yield f"a:{json.dumps(_tool_call_result)}\n"
+                    elif event.type == "agent_updated_stream_event":
+                        logger.debug(
+                            "[stream_data_async] Agent switched to: %s", event.new_agent.name
+                        )
+
+            except Exception as e:  # pylint: disable=broad-except
+                logger.exception("Error in stream_data_async")
+                yield f"3:{json.dumps(str(e))}\n"
+                finish_reason = "error"
+
+            # At the end, save the response and yield the finish message part
+            _response_usage = Usage()
+            for raw_response in result.raw_responses:
+                _response_usage.add(raw_response.usage)
+
+            await sync_to_async(self._update_conversation)(
+                openai_messages, result.raw_responses, _response_usage
+            )
+
+            _finish_message = {
+                "finishReason": finish_reason,
+                "usage": {
+                    "promptTokens": _response_usage.input_tokens,
+                    "completionTokens": _response_usage.output_tokens,
+                },
+            }
+            yield f"d:{json.dumps(_finish_message)}\n"
+
+    def _update_conversation(
+        self,
+        input_messages: List[ResponseInputItemParam],
+        result_raw_responses: List[ModelResponse],
+        response_usage: Usage,
+    ):
+        ui_messages = []
+
+        self.conversation.openai_messages = input_messages + [
+            output.model_dump()
+            for raw_response in result_raw_responses
+            for output in raw_response.output
+        ]
+
+        for raw_response in result_raw_responses:
+            ui_messages += self._convert_openai_output_to_ui_messages(raw_response.output)
+
+        self.conversation.messages = self.conversation.ui_messages + [
+            ui_message.model_dump() for ui_message in ui_messages
+        ]
+
+        if self.conversation.agent_usage:
+            total_usage = ResponseUsage(**self.conversation.agent_usage)
+        else:
+            total_usage = ResponseUsage(
+                input_tokens=0,
+                output_tokens=0,
+                input_tokens_details=InputTokensDetails(cached_tokens=0),
+                output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
+                total_tokens=0,
+            )
+
+        total_usage.input_tokens += response_usage.input_tokens  # pylint: disable=no-member
+        total_usage.output_tokens += response_usage.output_tokens  # pylint: disable=no-member
+        total_usage.input_tokens_details.cached_tokens += (
+            response_usage.input_tokens_details.cached_tokens
+        )
+        total_usage.output_tokens_details.reasoning_tokens += (
+            response_usage.output_tokens_details.reasoning_tokens
+        )
+        total_usage.total_tokens = response_usage.total_tokens
+
+        self.conversation.agent_usage = total_usage.model_dump()
+
+        self.conversation.save()
+
+    def _convert_openai_output_to_ui_messages(
+        self, output: List[ResponseOutputItem]
+    ) -> List[UIMessage]:
+        """Convert OpenAI output to UI messages."""
+        ui_messages = []
+
+        for item in output:
+            if item.type == "message":
+                text_parts = [TextUIPart(type="text", text=item.text) for item in item.content]
+                ui_messages.append(
+                    UIMessage(
+                        id=str(uuid.uuid4()),
+                        role="assistant",
+                        parts=text_parts,
+                        content="".join(part.text for part in text_parts),
+                    )
+                )
+            elif item.type == "function_call":
+                if item.status == "in_progress":
+                    tool_invocation = ToolInvocationPartialCall(
+                        state="partial-call",
+                        step=None,
+                        toolCallId=item.call_id,
+                        toolName=item.name,
+                        args=json.loads(item.arguments),
+                    )
+                elif item.status == "completed":
+                    tool_invocation = ToolInvocationResult(
+                        state="result",
+                        step=None,
+                        toolCallId=item.call_id,
+                        toolName=item.name,
+                        args=json.loads(item.arguments),
+                        result=json.loads(item.result) if item.result else None,
+                    )
+                # elif item.status == "incomplete":
+                else:
+                    logger.warning("[stream_data_async] Unhandled message: %s", item)
+                    continue
+
+                ui_tool_invocation = ToolInvocationUIPart(
+                    type="tool-invocation",
+                    toolInvocation=tool_invocation,
+                )
+                ui_messages.append(UIMessage(role="assistant", parts=[ui_tool_invocation]))
+            # Handle other types as needed
+        return ui_messages

src/backend/chat/clients/pydantic_ai.py

@@ -1,791 +0,0 @@
-"""
-Pydantic-AI based AIAgentService.
-
-This file replaces the previous OpenAI-specific client with a Pydantic-AI
-implementation while keeping the *exact* same public API so that no
-changes are needed in views.py or tests.
-"""
-
-import dataclasses
-import json
-import logging
-import time
-import uuid
-from contextlib import AsyncExitStack, ExitStack
-from io import BytesIO
-from typing import Dict, List, Optional, Tuple
-
-from django.conf import settings
-from django.contrib.auth import get_user_model
-from django.core.cache import cache
-from django.core.exceptions import ImproperlyConfigured
-from django.core.files.storage import default_storage
-from django.db.models import Q
-from django.utils.module_loading import import_string
-
-from asgiref.sync import sync_to_async
-from langfuse import get_client
-from pydantic_ai import Agent
-from pydantic_ai.messages import (
-    BinaryContent,
-    DocumentUrl,
-    FunctionToolCallEvent,
-    FunctionToolResultEvent,
-    ImageUrl,
-    ModelMessage,
-    ModelMessagesTypeAdapter,
-    ModelRequest,
-    ModelResponse,
-    PartDeltaEvent,
-    PartStartEvent,
-    RetryPromptPart,
-    TextPart,
-    TextPartDelta,
-    ThinkingPart,
-    ThinkingPartDelta,
-    ToolCallPart,
-    ToolCallPartDelta,
-    ToolReturn,
-    ToolReturnPart,
-    UserPromptPart,
-)
-
-from core.feature_flags.helpers import is_feature_enabled
-from core.file_upload.utils import generate_retrieve_policy
-
-from chat import models
-from chat.agents.conversation import ConversationAgent
-from chat.agents.local_media_url_processors import (
-    update_history_local_urls,
-    update_local_urls,
-)
-from chat.agents.summarize import hand_off_to_summarization_agent
-from chat.ai_sdk_types import (
-    LanguageModelV1Source,
-    SourceUIPart,
-    UIMessage,
-)
-from chat.clients.async_to_sync import convert_async_generator_to_sync
-from chat.clients.exceptions import StreamCancelException
-from chat.clients.pydantic_ui_message_converter import (
-    model_message_to_ui_message,
-    ui_message_to_user_content,
-)
-from chat.mcp_servers import get_mcp_servers
-from chat.tools.document_search_rag import add_document_rag_search_tool
-from chat.vercel_ai_sdk.core import events_v4, events_v5
-from chat.vercel_ai_sdk.encoder import EventEncoder
-
-logger = logging.getLogger(__name__)
-
-User = get_user_model()
-
-
-@dataclasses.dataclass
-class ContextDeps:
-    """Dependencies for context management."""
-
-    conversation: models.ChatConversation
-    user: User
-    web_search_enabled: bool = False
-
-
-def get_model_configuration(model_hrid: str):
-    """Get the model configuration from settings."""
-    try:
-        return settings.LLM_CONFIGURATIONS[model_hrid]
-    except KeyError as exc:
-        raise ImproperlyConfigured(f"LLM model configuration '{model_hrid}' not found.") from exc
-
-
-class AIAgentService:  # pylint: disable=too-many-instance-attributes
-    """Service class for AI-related operations (Pydantic-AI edition)."""
-
-    def __init__(self, conversation: models.ChatConversation, user, model_hrid=None, language=None):
-        """
-        Initialize the AI agent service.
-
-        Args:
-            conversation: The chat conversation instance
-            user: The authenticated user instance, only used for dynamic feature flags
-        """
-        self.conversation = conversation
-        self.user = user  # authenticated user only
-        self.model_hrid = model_hrid or settings.LLM_DEFAULT_MODEL_HRID  # HRID of the model to use
-        self.language = language  # might be None
-        self._last_stop_check = 0
-
-        self._store_analytics = settings.LANGFUSE_ENABLED and user.allow_conversation_analytics
-        self.event_encoder = EventEncoder("v4")  # Always use v4 for now
-
-        self._support_streaming = True
-        if (streaming := get_model_configuration(self.model_hrid).supports_streaming) is not None:
-            self._support_streaming = streaming
-
-        # Feature flags
-        self._is_document_upload_enabled = is_feature_enabled(self.user, "document_upload")
-        self._is_web_search_enabled = is_feature_enabled(self.user, "web_search")
-        self._fake_streaming_delay = settings.FAKE_STREAMING_DELAY
-
-        self._context_deps = ContextDeps(
-            conversation=conversation,
-            user=user,
-            web_search_enabled=self._is_web_search_enabled,
-        )
-
-        self.conversation_agent = ConversationAgent(
-            model_hrid=self.model_hrid,
-            language=self.language,
-            instrument=self._store_analytics,
-            deps_type=ContextDeps,
-        )
-
-    @property
-    def _stop_cache_key(self):
-        return f"streaming:stop:{self.conversation.pk}"
-
-    # --------------------------------------------------------------------- #
-    # Public streaming API (unchanged signatures)
-    # --------------------------------------------------------------------- #
-
-    def stream_text(self, messages: List[UIMessage], force_web_search: bool = False):
-        """Return only the assistant text deltas (legacy text mode)."""
-        return convert_async_generator_to_sync(self.stream_text_async(messages, force_web_search))
-
-    def stream_data(self, messages: List[UIMessage], force_web_search: bool = False):
-        """Return Vercel-AI-SDK formatted events."""
-        return convert_async_generator_to_sync(self.stream_data_async(messages, force_web_search))
-
-    def stop_streaming(self):
-        """
-        Stop the current streaming operation.
-
-        This method is a placeholder for stopping the streaming operation.
-        """
-        logger.info("Stopping streaming for conversation %s", self.conversation.id)
-        cache.set(self._stop_cache_key, "1", timeout=30 * 60)  # 30 minutes timeout
-
-    # --------------------------------------------------------------------- #
-    # Async internals
-    # --------------------------------------------------------------------- #
-
-    async def stream_text_async(self, messages: List[UIMessage], force_web_search: bool = False):
-        """Return only the assistant text deltas (legacy text mode)."""
-        await self._clean()
-        with ExitStack() as stack:
-            if self._store_analytics:
-                span = stack.enter_context(get_client().start_as_current_span(name="conversation"))
-                span.update_trace(user_id=str(self.user.sub), session_id=str(self.conversation.pk))
-
-            async for event in self._run_agent(messages, force_web_search):
-                if stream_text := self.event_encoder.encode_text(event):
-                    yield stream_text
-
-    async def stream_data_async(self, messages: List[UIMessage], force_web_search: bool = False):
-        """Return Vercel-AI-SDK formatted events."""
-        await self._clean()
-        with ExitStack() as stack:
-            if self._store_analytics:
-                span = stack.enter_context(get_client().start_as_current_span(name="conversation"))
-                span.update_trace(user_id=str(self.user.sub), session_id=str(self.conversation.pk))
-            async for event in self._run_agent(messages, force_web_search):
-                if stream_data := self.event_encoder.encode(event):
-                    yield stream_data
-
-    async def _agent_stop_streaming(self, force_cache_check: Optional[bool] = False) -> None:
-        """Check if the agent should stop streaming."""
-        now = time.time()  # Current time in seconds since epoch
-
-        # Check if we should skip the cache check to avoid frequent checks
-        # This is useful to avoid unnecessary cache checks during streaming
-        # Check every 2 seconds
-        if not force_cache_check and now - self._last_stop_check < 2:
-            return
-        self._last_stop_check = now
-
-        if await cache.aget(self._stop_cache_key):
-            logger.info("Streaming stopped by cache key for conversation %s", self.conversation.id)
-            await cache.adelete(self._stop_cache_key)
-            raise StreamCancelException()
-        return
-
-    async def _clean(self):
-        """
-        Clean up the agent service.
-
-        This method is called when the agent service is no longer needed.
-        It can be used to release resources or perform any necessary cleanup.
-        """
-        self._last_stop_check = 0
-        await cache.adelete(self._stop_cache_key)
-
-    # --------------------------------------------------------------------- #
-    # Core agent runner
-    # --------------------------------------------------------------------- #
-    async def parse_input_documents(self, documents: List[BinaryContent | DocumentUrl]):
-        """
-        Parse and store input documents in the conversation's document store.
-        """
-        # Early external document URL rejection
-        if any(
-            not document.url.startswith("/media-key/")
-            for document in documents
-            if isinstance(document, DocumentUrl)
-        ):
-            raise ValueError("External document URL are not accepted yet.")
-        if any(
-            not document.url.startswith(f"/media-key/{self.conversation.pk}/")
-            for document in documents
-            if isinstance(document, DocumentUrl)
-        ):
-            raise ValueError("Document URL does not belong to the conversation.")
-
-        document_store_backend = import_string(settings.RAG_DOCUMENT_SEARCH_BACKEND)
-
-        document_store = document_store_backend(self.conversation.collection_id)
-        if not document_store.collection_id:
-            # Create a new collection for the conversation
-            collection_id = document_store.create_collection(
-                name=f"conversation-{self.conversation.pk}",
-            )
-            self.conversation.collection_id = str(collection_id)
-            await self.conversation.asave(update_fields=["collection_id", "updated_at"])
-
-        for document in documents:
-            key = None
-            if isinstance(document, DocumentUrl):
-                if document.url.startswith("/media-key/"):
-                    # Local file, retrieve from object storage
-                    key = document.url[len("/media-key/") :]
-                    # Security check: ensure the document belongs to the conversation
-                    if not key.startswith(f"{self.conversation.pk}/"):
-                        raise ValueError("Document URL does not belong to the conversation.")
-                    # Retrieve the document data
-                    with default_storage.open(key, "rb") as file:
-                        document_data = file.read()
-                    parsed_content = document_store.parse_and_store_document(
-                        name=document.identifier,
-                        content_type=document.media_type,
-                        content=document_data,
-                    )
-                else:
-                    # Remote URL
-                    raise ValueError("External document URL are not accepted yet.")
-            else:
-                parsed_content = document_store.parse_and_store_document(
-                    name=document.identifier,
-                    content_type=document.media_type,
-                    content=document.data,
-                )
-
-            if not document.media_type.startswith("text/"):
-                md_attachment = await models.ChatConversationAttachment.objects.acreate(
-                    conversation=self.conversation,
-                    uploaded_by=self.user,
-                    key=key or f"{self.conversation.pk}/attachments/{document.identifier}.md",
-                    file_name=f"{document.identifier}.md",
-                    content_type="text/markdown",
-                    conversion_from=key,  # might be None
-                )
-                default_storage.save(md_attachment.key, BytesIO(parsed_content.encode("utf8")))
-                md_attachment.upload_state = models.AttachmentStatus.READY
-                await md_attachment.asave(update_fields=["upload_state", "updated_at"])
-
-    def prepare_prompt(  # noqa: PLR0912  # pylint: disable=too-many-branches
-        self, message: UIMessage
-    ) -> Tuple[str, List[BinaryContent | ImageUrl], List[BinaryContent]]:
-        """
-        Prepare the user prompt for the agent.
-
-        This method is used to convert a UIMessage into a format suitable for the agent.
-        It extracts the user content from the message and returns it as a list of UserContent.
-        """
-        user_content = ui_message_to_user_content(message)
-
-        user_prompt = []
-        attachment_images = []
-        attachment_documents = []
-        attachment_audio = []
-        attachment_video = []
-        for content in user_content:
-            if isinstance(content, str):
-                user_prompt.append(content)
-            elif isinstance(content, BinaryContent):
-                if content.is_audio:
-                    attachment_audio.append(content)
-                elif content.is_video:
-                    attachment_video.append(content)
-                elif content.is_image:
-                    attachment_images.append(content)
-                else:
-                    attachment_documents.append(content)
-            elif isinstance(content, ImageUrl):
-                attachment_images.append(content)
-            elif isinstance(content, DocumentUrl):
-                attachment_documents.append(content)
-            else:
-                # Should never happen, but just in case
-                raise ValueError(f"Unsupported UserContent type: {type(content)}")
-
-        if any(attachment_audio):
-            # Should be handled by the frontend, but just in case
-            raise ValueError("Audio attachments are not supported in the current implementation.")
-        if any(attachment_video):
-            # Should be handled by the frontend, but just in case
-            raise ValueError("Video attachments are not supported in the current implementation.")
-
-        if len(user_prompt) != 1:
-            raise ValueError(
-                "User prompt must contain exactly one text part, "
-                f"but got {len(user_prompt)} parts: {user_prompt}"
-            )
-
-        return user_prompt[0], attachment_images, attachment_documents
-
-    async def _run_agent(  # noqa: PLR0912, PLR0915 # pylint: disable=too-many-branches,too-many-statements, too-many-locals, too-many-return-statements
-        self,
-        messages: List[UIMessage],
-        force_web_search: bool = False,
-    ) -> events_v4.Event | events_v5.Event:
-        """Run the Pydantic AI agent and stream events."""
-        if messages[-1].role != "user":
-            return
-
-        # Langfuse settings
-        if self._store_analytics:
-            langfuse = get_client()
-            langfuse.update_current_trace(
-                session_id=str(self.conversation.pk),
-                user_id=str(self.user.sub),
-                metadata={
-                    "user_fqdn": self.user.email.split("@")[-1],  # no need for security here
-                },
-            )
-
-        history = ModelMessagesTypeAdapter.validate_python(self.conversation.pydantic_messages)
-        history = update_history_local_urls(
-            self.conversation, history
-        )  # presign URLs for local images
-
-        user_prompt, input_images, input_documents = self.prepare_prompt(messages[-1])
-
-        image_key_mapping = {}
-        if input_images:
-            # presign URLs for local images
-            input_images = update_local_urls(
-                self.conversation, input_images, updated_url=image_key_mapping
-            )
-
-        if self._store_analytics:
-            langfuse.update_current_trace(input=user_prompt)
-
-        usage = {"promptTokens": 0, "completionTokens": 0}
-
-        conversation_has_documents = self._is_document_upload_enabled and (
-            bool(self.conversation.collection_id)
-            or bool(
-                await models.ChatConversationAttachment.objects.filter(
-                    conversation=self.conversation,
-                    content_type__startswith="text/",
-                ).aexists()
-            )
-        )
-
-        if not self._is_document_upload_enabled and input_documents:
-            logger.warning("Document upload feature is disabled, ignoring input documents.")
-            input_documents = []
-
-        if input_documents:
-            _tool_call_id = str(uuid.uuid4())
-            yield events_v4.ToolCallPart(
-                tool_call_id=_tool_call_id,
-                tool_name="document_parsing",
-                args={
-                    "documents": [
-                        {
-                            "identifier": doc.identifier,
-                        }
-                        for doc in input_documents
-                    ],
-                },
-            )
-            try:
-                await self.parse_input_documents(input_documents)
-            except Exception as exc:  # pylint: disable=broad-except
-                logger.exception("Error parsing input documents: %s", exc)
-                yield events_v4.ToolResultPart(
-                    tool_call_id=_tool_call_id,
-                    result={"state": "error", "error": str(exc)},
-                )
-                yield events_v4.FinishMessagePart(
-                    finish_reason=events_v4.FinishReason.ERROR,
-                    usage=events_v4.Usage(
-                        prompt_tokens=usage["promptTokens"],
-                        completion_tokens=usage["completionTokens"],
-                    ),
-                )
-                return
-            if not conversation_has_documents:
-                conversation_has_documents = True
-
-            yield events_v4.ToolResultPart(
-                tool_call_id=_tool_call_id,
-                result={"state": "done"},
-            )
-
-        await self._agent_stop_streaming(force_cache_check=True)
-
-        if force_web_search and not self._is_web_search_enabled:
-            logger.warning("Web search is forced but the feature is disabled, ignoring.")
-            force_web_search = False
-
-        web_search_tool_name = self.conversation_agent.get_web_search_tool_name()
-        if force_web_search and not web_search_tool_name:
-            logger.warning("Web search is forced but no web search tool is available, ignoring.")
-            force_web_search = False
-
-        if force_web_search:
-
-            @self.conversation_agent.system_prompt
-            def force_web_search_prompt() -> str:
-                """Dynamic system prompt function to force web search."""
-                return (
-                    f"You must call the {web_search_tool_name} tool "
-                    "before answering the user request."
-                )
-
-        _tool_is_streaming = False
-        _model_response_message_id = None
-
-        # Check for existing non-PDF documents in the conversation:
-        # - if no document at all: do nothing
-        # - if only PDFs: prepare document URLs for the agent
-        # - if other document types: add the RAG search tool
-        #   to allow searching in all kinds of documents
-        has_not_pdf_docs = await (
-            models.ChatConversationAttachment.objects.filter(
-                Q(conversion_from__isnull=True) | Q(conversion_from=""),
-                conversation=self.conversation,
-            )
-            .exclude(
-                Q(content_type__startswith="image/") | Q(content_type="application/pdf"),
-            )
-            .aexists()
-        )
-
-        document_urls = []
-        if not conversation_has_documents and not has_not_pdf_docs:
-            # No documents to process
-            pass
-        elif has_not_pdf_docs:
-            add_document_rag_search_tool(self.conversation_agent)
-
-            @self.conversation_agent.system_prompt
-            def summarization_system_prompt() -> str:
-                return (
-                    "When you receive a result from the summarization tool, you MUST return it "
-                    "directly to the user without any modification, paraphrasing, or additional "
-                    "summarization."
-                    "The tool already produces optimized summaries that should be presented "
-                    "verbatim."
-                    "You may translate the summary if required, but you MUST preserve all the "
-                    "information from the original summary."
-                    "You may add a follow-up question after the summary if needed."
-                )
-
-            @self.conversation_agent.tool
-            async def summarize(ctx) -> ToolReturn:
-                """
-                Summarize the documents for the user, only when asked for,
-                the documents are in my context.
-                """
-                return await hand_off_to_summarization_agent(ctx)
-        else:
-            conversation_documents = [
-                cd
-                async for cd in models.ChatConversationAttachment.objects.filter(
-                    Q(conversion_from__isnull=True) | Q(conversion_from=""),
-                    conversation=self.conversation,
-                )
-                .exclude(
-                    content_type__startswith="image/",
-                )
-                .values_list("key", "content_type")
-            ]
-
-            for doc_key, doc_content_type in conversation_documents:
-                if doc_content_type == "application/pdf":
-                    _presigned_url = generate_retrieve_policy(doc_key)
-                    document_urls.append(
-                        DocumentUrl(
-                            url=_presigned_url,
-                            identifier=doc_key.split("/")[-1],
-                            media_type="application/pdf",
-                        )
-                    )
-                    image_key_mapping[_presigned_url] = f"/media-key/{doc_key}"
-
-        async with AsyncExitStack() as stack:
-            # MCP servers (if any) can be initialized here
-            mcp_servers = [await stack.enter_async_context(mcp) for mcp in get_mcp_servers()]
-
-            _final_output_from_tool = None
-            _ui_sources = []
-
-            # Help Mistral to prevent `Unexpected role 'user' after role 'tool'` error.
-            if history and history[-1].kind == "request":
-                if history[-1].parts[-1].part_kind == "tool-return":
-                    history.append(ModelResponse(parts=[TextPart(content="ok")], kind="response"))
-
-            async with self.conversation_agent.iter(
-                [user_prompt] + input_images + document_urls,
-                message_history=history,  # history will pass through agent's history_processors
-                deps=self._context_deps,
-                toolsets=mcp_servers,
-            ) as run:
-                async for node in run:
-                    await self._agent_stop_streaming()
-                    if Agent.is_user_prompt_node(node):
-                        # A user prompt node => The user has provided input
-                        pass
-
-                    elif Agent.is_model_request_node(node):  # pylint: disable=too-many-nested-blocks
-                        # A model request node => agent is asking the model to generate a response
-                        if not self._support_streaming:
-                            result = await node.run(run.ctx)
-                            logger.debug("node.run result: %s", result)
-                            for part in result.model_response.parts:
-                                if isinstance(part, TextPart):
-                                    if self._fake_streaming_delay:
-                                        for i in range(0, len(part.content), 4):
-                                            await self._agent_stop_streaming()
-                                            yield events_v4.TextPart(text=part.content[i : i + 4])
-                                            time.sleep(self._fake_streaming_delay)
-                                    else:
-                                        yield events_v4.TextPart(text=part.content)
-                                elif isinstance(part, ToolCallPart):
-                                    yield events_v4.ToolCallPart(
-                                        tool_call_id=part.tool_call_id,
-                                        tool_name=part.tool_name,
-                                        args=json.loads(part.args) if part.args else {},
-                                    )
-                                elif isinstance(part, ThinkingPart):
-                                    yield events_v4.ReasoningPart(reasoning=part.content)
-                                else:
-                                    logger.warning(
-                                        "Unknown part type in model response: %s %s",
-                                        type(part),
-                                        dataclasses.asdict(part),
-                                    )
-                            continue
-
-                        async with node.stream(run.ctx) as request_stream:
-                            async for event in request_stream:
-                                await self._agent_stop_streaming()
-                                logger.debug("Received request_stream event: %s", type(event))
-                                if isinstance(event, PartStartEvent):
-                                    logger.debug("PartStartEvent: %s", dataclasses.asdict(event))
-
-                                    if isinstance(event.part, TextPart):
-                                        yield events_v4.TextPart(text=event.part.content)
-                                    elif isinstance(event.part, ToolCallPart):
-                                        yield events_v4.ToolCallStreamingStartPart(
-                                            tool_call_id=event.part.tool_call_id,
-                                            tool_name=event.part.tool_name,
-                                        )
-                                    elif isinstance(event.part, ThinkingPart):
-                                        yield events_v4.ReasoningPart(
-                                            reasoning=event.part.content,
-                                        )
-
-                                elif isinstance(event, PartDeltaEvent):
-                                    logger.debug(
-                                        "PartDeltaEvent: %s %s",
-                                        type(event),
-                                        dataclasses.asdict(event),
-                                    )
-                                    if isinstance(event.delta, TextPartDelta):
-                                        yield events_v4.TextPart(text=event.delta.content_delta)
-                                    elif isinstance(event.delta, ToolCallPartDelta):
-                                        _tool_is_streaming = True
-                                        yield events_v4.ToolCallDeltaPart(
-                                            tool_call_id=event.delta.tool_call_id,
-                                            args_text_delta=event.delta.args_delta,
-                                        )
-                                    elif isinstance(event.delta, ThinkingPartDelta):
-                                        yield events_v4.ReasoningPart(
-                                            reasoning=event.delta.content_delta,
-                                        )
-
-                    elif Agent.is_call_tools_node(node):
-                        # A handle-response node => The model returned some data,
-                        # potentially calls a tool
-                        async with node.stream(run.ctx) as handle_stream:
-                            async for event in handle_stream:
-                                await self._agent_stop_streaming()
-                                logger.debug(
-                                    "Received request_stream event: %s, %s",
-                                    type(event),
-                                    dataclasses.asdict(event),
-                                )
-                                if isinstance(event, FunctionToolCallEvent):
-                                    if not _tool_is_streaming:
-                                        yield events_v4.ToolCallPart(
-                                            tool_call_id=event.tool_call_id,
-                                            tool_name=event.part.tool_name,
-                                            args=json.loads(event.part.args)
-                                            if event.part.args
-                                            else {},
-                                        )
-                                elif isinstance(event, FunctionToolResultEvent):
-                                    if isinstance(event.result, ToolReturnPart):
-                                        if event.result.metadata and (
-                                            sources := event.result.metadata.get("sources")
-                                        ):
-                                            for source_url in sources:
-                                                url_source = LanguageModelV1Source(
-                                                    sourceType="url",
-                                                    id=str(uuid.uuid4()),
-                                                    url=source_url,
-                                                    providerMetadata={},
-                                                )
-                                                _new_source_ui = SourceUIPart(
-                                                    type="source", source=url_source
-                                                )
-                                                _ui_sources.append(_new_source_ui)
-                                                yield events_v4.SourcePart(
-                                                    **_new_source_ui.source.model_dump()
-                                                )
-
-                                        yield events_v4.ToolResultPart(
-                                            tool_call_id=event.tool_call_id,
-                                            result=event.result.content,
-                                        )
-                                    elif isinstance(event.result, RetryPromptPart):
-                                        yield events_v4.ToolResultPart(
-                                            tool_call_id=event.tool_call_id,
-                                            result=event.result.content,
-                                        )
-                                    else:
-                                        logger.warning(
-                                            "Unexpected tool result type: %s %s",
-                                            type(event.result),
-                                            dataclasses.asdict(event.result),
-                                        )
-                    elif Agent.is_end_node(node):
-                        # Once an End node is reached, the agent run is complete
-                        logger.debug("v: %s", dataclasses.asdict(node))
-
-                        # Enforce the message ID to store the trace ID and allow scoring later
-                        # We use the start step part (to set the message ID) even if it's
-                        # not the purpose of this event, but Vercel AI SDK does not
-                        # have a better place to set the message ID and we only want to enforce
-                        # The last message to store the trace ID...
-                        if _model_response_message_id:
-                            logger.error("_model_response_message_id already set")
-                        _model_response_message_id = (
-                            str(uuid.uuid4())
-                            if not self._store_analytics
-                            else f"trace-{langfuse.get_current_trace_id()}"
-                        )
-                        yield events_v4.StartStepPart(
-                            message_id=_model_response_message_id,
-                        )
-
-                # Final usage summary
-                final_usage = run.usage()
-                usage["promptTokens"] = final_usage.input_tokens
-                usage["completionTokens"] = final_usage.output_tokens
-
-        await self._agent_stop_streaming(force_cache_check=True)
-
-        # Persist conversation
-        await sync_to_async(self._update_conversation)(
-            final_output=run.result.new_messages(),
-            usage=usage,
-            final_output_from_tool=_final_output_from_tool,
-            ui_sources=_ui_sources,
-            model_response_message_id=_model_response_message_id,
-            image_key_mapping=image_key_mapping or None,
-        )
-
-        if self._store_analytics:
-            langfuse.update_current_trace(output=run.result.output)
-
-        # Vercel finish message
-        yield events_v4.FinishMessagePart(
-            finish_reason=events_v4.FinishReason.STOP,
-            usage=events_v4.Usage(
-                prompt_tokens=usage["promptTokens"],
-                completion_tokens=usage["completionTokens"],
-            ),
-        )
-
-    def _update_conversation(  # noqa: PLR0913
-        self,
-        *,
-        final_output: List[ModelRequest | ModelMessage],
-        usage: Dict[str, int],
-        final_output_from_tool: str | None,
-        ui_sources: List[SourceUIPart] = None,
-        model_response_message_id: str | None = None,
-        image_key_mapping: Dict[str, str] = None,
-    ):  # pylint: disable=too-many-arguments
-        """
-        Save everything related to the conversation.
-
-        Things to improve here:
-         - The way we need to add the UI sources to the final output message.
-
-        Args:
-            final_output (List[ModelRequest | ModelMessage]): The final output from the agent.
-            usage (Dict[str, int]): The token usage statistics.
-            user_initial_prompt_str (str | None): The initial user prompt string, if any.
-            ui_sources (List[SourceUIPart]): Optional UI sources to include in the conversation.
-        """
-        _merged_final_output_request = ModelRequest(
-            parts=[
-                part for msg in final_output if isinstance(msg, ModelRequest) for part in msg.parts
-            ],
-            kind="request",
-        )
-        _merged_final_output_message = ModelResponse(
-            parts=[
-                part for msg in final_output if isinstance(msg, ModelResponse) for part in msg.parts
-            ]
-            + ([TextPart(content=final_output_from_tool)] if final_output_from_tool else []),
-            kind="response",
-        )
-
-        if image_key_mapping:
-            for part in _merged_final_output_request.parts:
-                if isinstance(part, UserPromptPart):
-                    for content in part.content:
-                        if isinstance(content, (ImageUrl, DocumentUrl)) and (
-                            unsigned_url := image_key_mapping.get(content.url)
-                        ):
-                            content.url = unsigned_url
-
-        _output_ui_message = model_message_to_ui_message(_merged_final_output_message)
-        if ui_sources:
-            _output_ui_message.parts += ui_sources
-        if model_response_message_id:
-            _output_ui_message.id = model_response_message_id
-        else:
-            logger.warning("model_response_message_id is None")
-
-        self.conversation.messages += [
-            model_message_to_ui_message(_merged_final_output_request),
-            _output_ui_message,
-        ]
-        self.conversation.agent_usage = usage
-
-        final_output_json = json.loads(
-            ModelMessagesTypeAdapter.dump_json(final_output).decode("utf-8")
-        )
-        logger.debug("final_output_json: %s", final_output_json)
-        self.conversation.pydantic_messages += json.loads(
-            ModelMessagesTypeAdapter.dump_json(final_output).decode("utf-8")
-        )
-
-        self.conversation.save()

src/backend/chat/clients/pydantic_ui_message_converter.py

@@ -1,252 +0,0 @@
-"""
-Utility functions to convert between UIMessage (ai_sdk_types.py)
-and UserContent/ModelMessage (pydantic_ai.messages.py).
-"""
-
-import base64
-import json
-import logging
-import uuid
-from dataclasses import asdict
-from typing import List
-
-from pydantic_ai.messages import (
-    BinaryContent,
-    DocumentUrl,
-    ImageUrl,
-    ModelMessage,
-    ModelRequest,
-    ModelResponse,
-    RetryPromptPart,
-    SystemPromptPart,
-    TextPart,
-    ThinkingPart,
-    ToolCallPart,
-    ToolReturnPart,
-    UserContent,
-    UserPromptPart,
-)
-
-from chat.ai_sdk_types import (
-    Attachment,
-    FileUIPart,
-    ReasoningDetailText,
-    ReasoningUIPart,
-    TextUIPart,
-    ToolInvocationCall,
-    ToolInvocationUIPart,
-    UIMessage,
-    UIPart,
-)
-
-
-def ui_message_to_user_content(message: UIMessage) -> List[UserContent]:
-    """
-    Convert a UIMessage to a list of UserContent for Pydantic-AI.
-    """
-    user_contents: List[UserContent] = []
-    for part in message.parts:
-        if isinstance(part, TextUIPart):
-            user_contents.append(part.text)
-        elif isinstance(part, FileUIPart):
-            user_contents.append(
-                BinaryContent(data=part.data.encode("utf-8"), media_type=part.mimeType)
-            )
-        elif isinstance(part, ToolInvocationUIPart):
-            # Tool invocations are not directly mapped to UserContent, skip or handle as needed
-            continue
-        elif isinstance(part, ReasoningUIPart):
-            # Reasoning parts are not directly mapped to UserContent, skip or handle as needed
-            continue
-        else:
-            raise ValueError(f"Unsupported UIPart type: {type(part)}")
-    for experimental_attachment in message.experimental_attachments or []:
-        if experimental_attachment.url.startswith("data:"):
-            # Handle data URLs
-            raw_data = base64.b64decode(experimental_attachment.url.split(",")[1])
-            user_contents.append(
-                BinaryContent(
-                    data=raw_data,
-                    media_type=experimental_attachment.contentType,
-                    identifier=experimental_attachment.name,
-                )
-            )
-        elif experimental_attachment.contentType.startswith("image/"):
-            user_contents.append(
-                ImageUrl(
-                    url=experimental_attachment.url,
-                    media_type=experimental_attachment.contentType,
-                    identifier=experimental_attachment.name,
-                )
-            )
-        else:
-            user_contents.append(
-                DocumentUrl(
-                    url=experimental_attachment.url,
-                    media_type=experimental_attachment.contentType,
-                    identifier=experimental_attachment.name,
-                )
-            )
-
-    return user_contents
-
-
-def model_message_to_ui_message(model_message: ModelMessage) -> UIMessage:  # noqa: PLR0912, PLR0915  # pylint: disable=too-many-statements
-    """
-    Convert a ModelMessage (ModelRequest or ModelResponse) to a UIMessage.
-    """
-    # pylint: disable=too-many-nested-blocks,too-many-branches
-    parts: List[UIPart] = []
-    experimental_attachments: List[Attachment] = []
-
-    logging.getLogger(__name__).debug(
-        "Converting ModelMessage to UIMessage: %s %s",
-        type(model_message),
-        asdict(model_message),
-    )
-    _states = {"tool-calls": {}}
-
-    if isinstance(model_message, ModelRequest):
-        message_timestamp = None
-
-        for part in model_message.parts:
-            if isinstance(part, SystemPromptPart):
-                # System prompts are not included in UIMessage parts
-                continue
-            if isinstance(part, UserPromptPart):
-                message_timestamp = part.timestamp
-                if isinstance(part.content, str):
-                    parts.append(TextUIPart(type="text", text=part.content))
-                elif isinstance(part.content, list):
-                    for c in part.content:
-                        if isinstance(c, str):
-                            parts.append(TextUIPart(type="text", text=c))
-                        elif isinstance(c, BinaryContent):
-                            experimental_attachments.append(
-                                Attachment(
-                                    contentType=c.media_type,
-                                    url=f"data:{c.media_type};base64,"
-                                    + base64.b64encode(c.data).decode("utf-8"),
-                                )
-                            )
-                        elif isinstance(c, ImageUrl):
-                            experimental_attachments.append(
-                                Attachment(
-                                    contentType=c.media_type,
-                                    url=c.url,
-                                    name=c.identifier,
-                                )
-                            )
-                        elif isinstance(c, DocumentUrl):
-                            experimental_attachments.append(
-                                Attachment(
-                                    contentType=c.media_type,
-                                    url=c.url,
-                                    name=c.identifier,
-                                )
-                            )
-                        else:  # AudioUrl, VideoUrl
-                            raise ValueError(
-                                f"Unsupported UserContent in UserPromptPart: {type(c)}"
-                            )
-            elif isinstance(part, TextPart) and part.content:
-                parts.append(TextUIPart(type="text", text=part.content))
-            elif isinstance(part, ToolReturnPart):
-                pass
-                # parts.append(ToolInvocationUIPart(
-                #     type="tool-invocation",
-                #     toolInvocation=ToolInvocationResult(
-                #         state="result",
-                #         toolCallId=part.tool_call_id,
-                #         toolName=part.tool_name,
-                #         args={},
-                #         result=part.content,
-                #     )
-                # ))
-            elif isinstance(part, ThinkingPart):
-                parts.append(
-                    ReasoningUIPart(
-                        type="reasoning",
-                        reasoning=part.content,
-                        details=[
-                            ReasoningDetailText(
-                                type="text",
-                                text=part.content,
-                                signature=part.signature,
-                            )
-                        ],
-                    )
-                )
-            elif isinstance(part, RetryPromptPart):
-                # Retry prompts are not included in UIMessage parts
-                continue
-            else:
-                raise ValueError(f"Unsupported ModelRequest part type: {type(part)}")
-
-        if not parts:
-            return None
-
-        return UIMessage(
-            id=str(uuid.uuid4()),
-            role="user",
-            content="".join(part.text for part in parts if isinstance(part, TextUIPart)),
-            parts=parts,
-            experimental_attachments=experimental_attachments or None,
-            createdAt=message_timestamp,
-        )
-
-    if isinstance(model_message, ModelResponse):
-        for part in model_message.parts:
-            if isinstance(part, UserPromptPart):
-                if isinstance(part.content, str):
-                    parts.append(TextUIPart(type="text", text=part.content))
-                elif isinstance(part.content, list):
-                    for c in part.content:
-                        if isinstance(c, str):
-                            parts.append(TextUIPart(type="text", text=c))
-                        else:  # ImageUrl, AudioUrl, VideoUrl, DocumentUrl, BinaryContent
-                            raise ValueError(
-                                f"Unsupported UserContent in UserPromptPart: {type(c)}"
-                            )
-            elif isinstance(part, TextPart):
-                parts.append(TextUIPart(type="text", text=part.content))
-            elif isinstance(part, ToolCallPart):
-                parts.append(
-                    ToolInvocationUIPart(
-                        type="tool-invocation",
-                        toolInvocation=ToolInvocationCall(
-                            state="call",
-                            toolCallId=part.tool_call_id,
-                            toolName=part.tool_name,
-                            args=json.loads(part.args)
-                            if isinstance(part.args, str)
-                            else part.args or {},
-                        ),
-                    )
-                )
-            elif isinstance(part, ThinkingPart):
-                parts.append(
-                    ReasoningUIPart(
-                        type="reasoning",
-                        reasoning=part.content,
-                        details=[
-                            ReasoningDetailText(
-                                type="text",
-                                text=part.content,
-                                signature=part.signature,
-                            )
-                        ],
-                    )
-                )
-            else:
-                raise ValueError(f"Unsupported ModelMessage part type: {type(part)}")
-
-        return UIMessage(
-            id=str(uuid.uuid4()),
-            role="assistant",
-            content="".join(part.text for part in parts if isinstance(part, TextUIPart)),
-            parts=parts,
-            createdAt=model_message.timestamp,
-        )
-
-    raise ValueError(f"Unsupported ModelMessage part type: {type(model_message)}")

src/backend/chat/factories.py

@@ -1,33 +0,0 @@
-"""Factories for chat application."""
-
-from uuid import uuid4
-
-import factory.django
-
-from core.factories import UserFactory
-
-from . import models
-
-
-class ChatConversationFactory(factory.django.DjangoModelFactory):
-    """Factory for creating ChatConversation instances."""
-
-    owner = factory.SubFactory(UserFactory)
-
-    class Meta:
-        model = models.ChatConversation
-
-
-class ChatConversationAttachmentFactory(factory.django.DjangoModelFactory):
-    """Factory for creating ChatConversationAttachment instances."""
-
-    conversation = factory.SubFactory(ChatConversationFactory)
-    uploaded_by = factory.SubFactory(UserFactory)
-    key = factory.LazyAttribute(
-        lambda obj: f"{obj.conversation.pk}/attachments/{uuid4()}.{obj.file_name.split('.')[-1]}"
-    )
-    file_name = factory.Faker("file_name")
-    content_type = factory.Faker("mime_type")
-
-    class Meta:
-        model = models.ChatConversationAttachment

src/backend/chat/llm_configuration.py

@@ -1,198 +0,0 @@
-"""Module for managing LLM configurations from a JSON configuration file."""
-
-import os
-from functools import lru_cache
-from typing import Annotated, Any, Literal, Optional, Self, Sequence
-
-from pydantic import (
-    AfterValidator,
-    BaseModel,
-    BeforeValidator,
-    Field,
-    ImportString,
-    field_validator,
-    model_validator,
-)
-from pydantic_ai.profiles import JsonSchemaTransformer
-
-
-def _get_setting_or_env_or_value(value: str) -> Any:
-    """Get the value from environment variable, Django settings, or return the value as is."""
-    from django.conf import settings  # pylint: disable=import-outside-toplevel # noqa: PLC0415
-
-    if value.startswith("environ."):
-        env_var = value.split("environ.")[1]
-        new_value = os.environ.get(env_var, None)
-        if new_value is None:
-            raise ValueError(f"Environment variable '{env_var}' not set.")
-        return new_value
-
-    if value.startswith("settings."):
-        setting_var = value.split("settings.")[1]
-        new_value = getattr(settings, setting_var, None)
-        if new_value is None:
-            raise ValueError(f"Django setting '{setting_var}' not set.")
-        return new_value
-
-    return value
-
-
-SettingEnvValue = Annotated[
-    str,
-    AfterValidator(_get_setting_or_env_or_value),
-]
-
-LongStringAsListValue = Annotated[
-    str,
-    BeforeValidator(lambda v: "".join(v) if isinstance(v, list) else v),
-]
-
-
-class LLMProvider(BaseModel):
-    """Model representing a provider of Large Language Models (LLMs)."""
-
-    hrid: str
-    base_url: SettingEnvValue
-    api_key: SettingEnvValue
-    kind: Literal["openai", "mistral"] = "openai"
-
-
-class LLMProfile(BaseModel):
-    """Based on pydantic_ai.profiles.ModelProfile to allow customization."""
-
-    supports_tools: bool | None = None
-    supports_json_schema_output: bool | None = None
-    supports_json_object_output: bool | None = None
-    default_structured_output_mode: str | None = None
-    prompted_output_template: str | None = None
-    json_schema_transformer: ImportString | None = Field(default=None, validate_default=True)
-    thinking_tags: tuple[str, str] | None = None
-    ignore_streamed_leading_whitespace: bool | None = None
-
-    # openai specific settings: should find a way to auto declare these
-    # based on OpenAIModelProfile.
-    openai_supports_strict_tool_definition: bool | None = None
-    openai_unsupported_model_settings: Sequence[str] | None = None
-    openai_supports_tool_choice_required: bool | None = None
-    openai_system_prompt_role: str | None = None
-    openai_chat_supports_web_search: bool | None = None
-    openai_supports_encrypted_reasoning_content: bool | None = None
-
-    @field_validator("json_schema_transformer", mode="after")
-    @classmethod
-    def validate_json_schema_transformer(
-        cls, value: JsonSchemaTransformer | None
-    ) -> Optional[JsonSchemaTransformer]:
-        """Convert the tools if it's a setting or environment variable."""
-        if not value:
-            return None
-
-        if issubclass(value, JsonSchemaTransformer):
-            return value
-
-        raise ValueError(f"Invalid JSON Schema Transformer '{value}'")
-
-
-class LLMSettings(BaseModel):
-    """Based on pydantic_ai.settings.ModelSettings to allow customization."""
-
-    max_tokens: int | None = None
-    temperature: float | None = None
-    top_p: float | None = None
-    timeout: float | None = None
-    parallel_tool_calls: bool | None = None
-    seed: int | None = None
-    presence_penalty: float | None = None
-    frequency_penalty: float | None = None
-    logit_bias: dict[str, int] | None = None
-    stop_sequences: list[str] | None = None
-    extra_headers: dict[str, str] | None = None
-    extra_body: dict[str, str] | None = None
-
-
-class LLModel(BaseModel):
-    """Model representing a Large Language Model (LLM)."""
-
-    hrid: str
-    model_name: SettingEnvValue
-    human_readable_name: str
-    profile: LLMProfile | None = None
-    provider_name: str | None = None
-    provider: LLMProvider | None = None
-    settings: LLMSettings | None = None
-    is_active: bool
-    icon: LongStringAsListValue | None = None
-    supports_streaming: bool | None = None
-    system_prompt: SettingEnvValue
-    tools: list[str]
-
-    @field_validator("tools", mode="before")
-    @classmethod
-    def validate_tools(cls, value: list[str] | str) -> list[str]:
-        """Convert the tools if it's a setting or environment variable."""
-        if isinstance(value, str):
-            return _get_setting_or_env_or_value(value)
-        return value
-
-    @model_validator(mode="after")
-    def check_provider_or_provider_name(self) -> Self:
-        """
-        Do some validation regarding provider and provider_name:
-        - Either `provider_name` or `provider` must be set, but not both.
-        - If neither is set, `model_name` must be in the format '<provider>:<model>'.
-        """
-        if bool(self.provider_name) and bool(self.provider):
-            raise ValueError("Either 'provider_name' or 'provider' must be set, but not both.")
-        if not self.provider_name and not self.provider and len(self.model_name.split(":")) != 2:
-            raise ValueError(
-                "Either 'provider_name' or 'provider' must be set, "
-                "unless model_name starts with '<provider>:'."
-            )
-        return self
-
-    @property
-    def is_custom(self) -> bool:
-        """Return True if the model is a custom model (i.e., defines a provider)."""
-        return self.provider is not None
-
-
-class LLMConfiguration(BaseModel):
-    """Model representing the entire LLM configuration."""
-
-    models: list[LLModel]
-    providers: list[LLMProvider]
-
-    @model_validator(mode="after")
-    def fill_providers(self) -> Self:
-        """Fill in the `provider` field of each model based on `provider_name`."""
-        provider_map = {provider.hrid: provider for provider in self.providers}
-        for model in self.models:
-            if model.provider_name:
-                try:
-                    model.provider = provider_map[model.provider_name]
-                except KeyError as exc:
-                    raise ValueError(
-                        f"Provider '{model.provider_name}' not found "
-                        f"for model '{model.model_name}'."
-                    ) from exc
-        return self
-
-
-def _read_llm_configuration(llm_configuration_file_path) -> LLMConfiguration:
-    """Read the LLM configuration from a JSON file and return an LLMConfiguration instance."""
-    with open(llm_configuration_file_path, "rb") as f:
-        data = f.read().decode("utf-8")
-    return LLMConfiguration.model_validate_json(data)
-
-
-def load_llm_configuration(llm_configuration_file_path) -> dict[str, LLModel]:
-    """Load the LLM configuration and return a mapping of model HRIDs to LLModel instances."""
-    configuration = _read_llm_configuration(llm_configuration_file_path)
-    model_map = {model.hrid: model for model in configuration.models}
-    return model_map
-
-
-@lru_cache(maxsize=1)
-def cached_load_llm_configuration(llm_configuration_file_path) -> dict[str, LLModel]:
-    """Load the LLM configuration with caching to avoid redundant loading."""
-    return load_llm_configuration(llm_configuration_file_path)

src/backend/chat/malware_detection.py

@@ -1,52 +0,0 @@
-"""Malware detection callbacks"""
-
-import logging
-
-from core.file_upload.enums import AttachmentStatus
-
-from chat.models import ChatConversationAttachment
-
-logger = logging.getLogger(__name__)
-security_logger = logging.getLogger("conversations.security")
-
-
-def conversation_safe_attachment_callback(file_path, *, conversation_id, **kwargs):
-    """Callback when a malware scan is completed and unsafe for a conversation attachment."""
-    logger.info("File %s for conversation %s is safe", file_path, conversation_id)
-
-    ChatConversationAttachment.objects.filter(
-        conversation_id=conversation_id, key=file_path
-    ).update(upload_state=AttachmentStatus.READY)
-
-
-def unknown_attachment_callback(file_path, error_info, *, conversation_id, **kwargs) -> bool:
-    """Callback when a malware scan is completed and unknown for a conversation attachment."""
-    security_logger.warning(
-        "File %s for conversation %s has an unknown reportstatus. Error info: %s",
-        file_path,
-        conversation_id,
-        error_info,
-    )
-
-    error_code = error_info.get("error_code")
-    if error_code == 413:
-        ChatConversationAttachment.objects.filter(
-            conversation_id=conversation_id, key=file_path
-        ).update(upload_state=AttachmentStatus.FILE_TOO_LARGE_TO_ANALYZE)
-        return True
-
-    return False
-
-
-def conversation_unsafe_attachment_callback(file_path, error_info, *, conversation_id, **kwargs):
-    """Callback when a malware scan is completed and unsafe for a conversation attachment."""
-    security_logger.warning(
-        "File %s for conversation %s is infected with malware. Error info: %s",
-        file_path,
-        conversation_id,
-        error_info,
-    )
-
-    ChatConversationAttachment.objects.filter(
-        conversation_id=conversation_id, key=file_path
-    ).update(upload_state=AttachmentStatus.SUSPICIOUS)

src/backend/chat/mcp_servers.py

@@ -1,6 +1,6 @@
 """MCP servers configuration: will be replaced by models."""
 
-from pydantic_ai.mcp import MCPServerStreamableHTTP
+from agents.mcp import MCPServerStreamableHttp, MCPServerStreamableHttpParams
 
 MCP_SERVERS = {
     "mcpServers": {
@@ -15,6 +15,9 @@ MCP_SERVERS = {
 def get_mcp_servers():
     """Retrieve MCP servers configuration."""
     return [
-        MCPServerStreamableHTTP(**server_config)
-        for _name, server_config in MCP_SERVERS["mcpServers"].items()
+        MCPServerStreamableHttp(
+            name=name,
+            params=MCPServerStreamableHttpParams(**server),
+        )
+        for name, server in MCP_SERVERS["mcpServers"].items()
     ]

src/backend/chat/migrations/0001_initial.py

@@ -1,24 +1,14 @@
-# Generated by Django 5.2.3 on 2025-08-06 16:42
+# Generated by Django 5.2.3 on 2025-06-26 12:15
 
 import uuid
 
-import django.core.serializers.json
-import django.db.models.deletion
-from django.conf import settings
 from django.db import migrations, models
 
-import django_pydantic_field.compat.django
-import django_pydantic_field.fields
-
-import chat.ai_sdk_types
-
 
 class Migration(migrations.Migration):
     initial = True
 
-    dependencies = [
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
+    dependencies = []
 
     operations = [
         migrations.CreateModel(
@@ -69,24 +59,19 @@ class Migration(migrations.Migration):
                     ),
                 ),
                 (
-                    "pydantic_messages",
+                    "openai_messages",
                     models.JSONField(
                         blank=True,
                         default=list,
-                        help_text="Pydantic messages for the chat conversation, used for history",
+                        help_text="OpenAI messages for the chat conversation, not used",
                     ),
                 ),
                 (
                     "messages",
-                    django_pydantic_field.fields.PydanticSchemaField(
+                    models.JSONField(
                         blank=True,
-                        config=None,
                         default=list,
-                        encoder=django.core.serializers.json.DjangoJSONEncoder,
                         help_text="Stored messages for the chat conversation, sent to frontend",
-                        schema=django_pydantic_field.compat.django.GenericContainer(
-                            list, (chat.ai_sdk_types.UIMessage,)
-                        ),
                     ),
                 ),
                 (
@@ -97,22 +82,6 @@ class Migration(migrations.Migration):
                         help_text="Agent usage for the chat conversation, provided by OpenAI API",
                     ),
                 ),
-                (
-                    "collection_id",
-                    models.CharField(
-                        blank=True,
-                        help_text="Collection ID for the conversation, used for RAG document search",
-                        null=True,
-                    ),
-                ),
-                (
-                    "owner",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="conversations",
-                        to=settings.AUTH_USER_MODEL,
-                    ),
-                ),
             ],
             options={
                 "abstract": False,

src/backend/chat/migrations/0002_fix_source_type_in_messages.py

@@ -1,54 +0,0 @@
-# Generated by Django 5.2.3 on 2025-09-15 11:47
-from django.db import migrations
-
-
-def forwards_func(apps, schema_editor):
-    """Use raw SQL to replace "source_type" with "sourceType" in the messages field."""
-    with schema_editor.connection.cursor() as cursor:
-        cursor.execute("""
-            UPDATE chat_chatconversation 
-            SET messages = REPLACE(messages::text, '"source_type"', '"sourceType"')::jsonb
-            WHERE messages::text LIKE '%"source_type"%'
-        """)
-    with schema_editor.connection.cursor() as cursor:
-        cursor.execute("""
-            UPDATE chat_chatconversation
-            SET ui_messages = REPLACE(ui_messages::text, '"source_type"', '"sourceType"')::jsonb
-            WHERE ui_messages::text LIKE '%"source_type"%'
-        """)
-
-
-def reverse_func(apps, schema_editor):
-    """Use raw SQL to replace "sourceType" with "source_type" in the messages field."""
-    with schema_editor.connection.cursor() as cursor:
-        cursor.execute("""
-            UPDATE chat_chatconversation 
-            SET messages = REPLACE(messages::text, '"sourceType"', '"source_type"')::jsonb
-            WHERE messages::text LIKE '%"sourceType"%'
-        """)
-    with schema_editor.connection.cursor() as cursor:
-        cursor.execute("""
-            UPDATE chat_chatconversation
-            SET ui_messages = REPLACE(ui_messages::text, '"sourceType"', '"source_type"')::jsonb
-            WHERE ui_messages::text LIKE '%"sourceType"%'
-        """)
-
-
-class Migration(migrations.Migration):
-    """
-    Rename source_type to sourceType in messages field of ChatConversation model.
-
-    Warning: This migration is not fail-safe, if the messages field contains
-    other occurrences of "source_type" or "sourceType" in other contexts, they will
-    also be replaced. Also, if the messages field is very large, this migration
-    may take a long time to run.
-    => OK because we are only in development phase.
-    """
-
-    dependencies = [
-        ("chat", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.RunPython(forwards_func, reverse_func),
-    ]

src/backend/chat/migrations/0002_initial.py

@@ -0,0 +1,26 @@
+# Generated by Django 5.2.3 on 2025-06-26 12:15
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = [
+        ("chat", "0001_initial"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="chatconversation",
+            name="owner",
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="conversations",
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

src/backend/chat/migrations/0003_chatconversationcontext.py

@@ -1,78 +0,0 @@
-# Generated by Django 5.2.3 on 2025-09-17 12:58
-
-import uuid
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("chat", "0002_fix_source_type_in_messages"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="ChatConversationContext",
-            fields=[
-                (
-                    "id",
-                    models.UUIDField(
-                        default=uuid.uuid4,
-                        editable=False,
-                        help_text="primary key for the record as UUID",
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="id",
-                    ),
-                ),
-                (
-                    "created_at",
-                    models.DateTimeField(
-                        auto_now_add=True,
-                        help_text="date and time at which a record was created",
-                        verbose_name="created on",
-                    ),
-                ),
-                (
-                    "updated_at",
-                    models.DateTimeField(
-                        auto_now=True,
-                        help_text="date and time at which a record was last updated",
-                        verbose_name="updated on",
-                    ),
-                ),
-                (
-                    "kind",
-                    models.CharField(
-                        choices=[("image", "Image"), ("document", "Document")],
-                        help_text="Kind of the chat conversation context (e.g., 'image', 'document')",
-                        max_length=50,
-                    ),
-                ),
-                (
-                    "name",
-                    models.CharField(
-                        help_text="Key of the chat conversation context", max_length=100
-                    ),
-                ),
-                (
-                    "content",
-                    models.TextField(
-                        blank=True, help_text="Value of the chat conversation context", null=True
-                    ),
-                ),
-                (
-                    "conversation",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="contexts",
-                        to="chat.chatconversation",
-                    ),
-                ),
-            ],
-            options={
-                "unique_together": {("conversation", "name")},
-            },
-        ),
-    ]

src/backend/chat/migrations/0004_chatconversationattachment_and_more.py

@@ -1,100 +0,0 @@
-# Generated by Django 5.2.7 on 2025-10-17 16:10
-
-import uuid
-
-import django.db.models.deletion
-from django.conf import settings
-from django.db import migrations, models
-
-import core.file_upload.enums
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("chat", "0003_chatconversationcontext"),
-        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="ChatConversationAttachment",
-            fields=[
-                (
-                    "id",
-                    models.UUIDField(
-                        default=uuid.uuid4,
-                        editable=False,
-                        help_text="primary key for the record as UUID",
-                        primary_key=True,
-                        serialize=False,
-                        verbose_name="id",
-                    ),
-                ),
-                (
-                    "created_at",
-                    models.DateTimeField(
-                        auto_now_add=True,
-                        help_text="date and time at which a record was created",
-                        verbose_name="created on",
-                    ),
-                ),
-                (
-                    "updated_at",
-                    models.DateTimeField(
-                        auto_now=True,
-                        help_text="date and time at which a record was last updated",
-                        verbose_name="updated on",
-                    ),
-                ),
-                (
-                    "upload_state",
-                    models.CharField(
-                        choices=core.file_upload.enums.AttachmentStatus.choices,
-                        default=core.file_upload.enums.AttachmentStatus["PENDING"],
-                        max_length=40,
-                    ),
-                ),
-                (
-                    "key",
-                    models.CharField(help_text="File path of the attachment in the object storage"),
-                ),
-                ("file_name", models.CharField(help_text="Original name of the attachment file")),
-                (
-                    "content_type",
-                    models.CharField(help_text="MIME type of the attachment file", max_length=100),
-                ),
-                ("size", models.PositiveBigIntegerField(blank=True, null=True)),
-                (
-                    "conversation",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="attachments",
-                        to="chat.chatconversation",
-                    ),
-                ),
-                (
-                    "uploaded_by",
-                    models.ForeignKey(
-                        help_text="User who uploaded the attachment",
-                        on_delete=django.db.models.deletion.PROTECT,
-                        related_name="uploaded_attachments",
-                        to=settings.AUTH_USER_MODEL,
-                    ),
-                ),
-                (
-                    "conversion_from",
-                    models.CharField(
-                        blank=True,
-                        help_text="Original file key if the Markdown from another file",
-                        null=True,
-                    ),
-                ),
-            ],
-            options={
-                "abstract": False,
-            },
-        ),
-        migrations.DeleteModel(
-            name="ChatConversationContext",
-        ),
-    ]

src/backend/chat/models.py

@@ -1,17 +1,10 @@
 """Models for chat conversations."""
 
-from typing import Sequence
-
 from django.contrib.auth import get_user_model
 from django.db import models
 
-from django_pydantic_field import SchemaField
-
-from core.file_upload.enums import AttachmentStatus
 from core.models import BaseModel
 
-from chat.ai_sdk_types import UIMessage
-
 User = get_user_model()
 
 
@@ -25,7 +18,7 @@ class ChatConversation(BaseModel):
       the 100 first characters of the first user input message.
     - `ui_messages`: A JSON field of UI messages sent by the frontend, all content is
       overridden at each new request from the frontend.
-    - `pydantic_messages`: A JSON field of PydanticAI messages, used to store conversation history.
+    - `openai_messages`: A JSON field of OpenAI messages, only for debug purpose, not used.
     - `messages`: A JSON field of stored messages for the conversation, sent to frontend
        when loading the conversation.
     - `agent_usage`: A JSON field of agent usage statistics for the conversation,
@@ -50,13 +43,12 @@ class ChatConversation(BaseModel):
         blank=True,
         help_text="UI messages for the chat conversation, sent by frontend, not used",
     )
-    pydantic_messages = models.JSONField(
+    openai_messages = models.JSONField(
         default=list,
         blank=True,
-        help_text="Pydantic messages for the chat conversation, used for history",
+        help_text="OpenAI messages for the chat conversation, not used",
     )
-    messages: Sequence[UIMessage] = SchemaField(
-        schema=list[UIMessage],
+    messages = models.JSONField(
         default=list,
         blank=True,
         help_text="Stored messages for the chat conversation, sent to frontend",
@@ -67,67 +59,3 @@ class ChatConversation(BaseModel):
         blank=True,
         help_text="Agent usage for the chat conversation, provided by OpenAI API",
     )
-
-    collection_id = models.CharField(
-        blank=True,
-        null=True,
-        help_text="Collection ID for the conversation, used for RAG document search",
-    )
-
-
-class ChatConversationAttachment(BaseModel):
-    """
-    Model representing an attachment associated with a chat conversation.
-
-    This model stores the details of an attachment:
-    - `conversation`: The conversation this attachment belongs to.
-    - `uploaded_by`: The user who uploaded the attachment.
-    - `key`: The file path of the attachment in the object storage.
-    - `file_name`: The original name of the attachment file.
-    - `content_type`: The MIME type of the attachment file.
-
-    """
-
-    conversation = models.ForeignKey(
-        ChatConversation,
-        related_name="attachments",
-        on_delete=models.CASCADE,
-        null=False,
-        blank=False,
-    )
-    uploaded_by = models.ForeignKey(
-        User,
-        related_name="uploaded_attachments",
-        on_delete=models.PROTECT,
-        null=False,
-        blank=False,
-        help_text="User who uploaded the attachment",
-    )
-    upload_state = models.CharField(
-        max_length=40,
-        choices=AttachmentStatus.choices,
-        default=AttachmentStatus.PENDING,
-    )
-    key = models.CharField(
-        blank=False,
-        null=False,
-        help_text="File path of the attachment in the object storage",
-    )
-    file_name = models.CharField(
-        blank=False,
-        null=False,
-        help_text="Original name of the attachment file",
-    )
-    content_type = models.CharField(
-        max_length=100,
-        blank=False,
-        null=False,
-        help_text="MIME type of the attachment file",
-    )
-    size = models.PositiveBigIntegerField(null=True, blank=True)
-
-    conversion_from = models.CharField(
-        blank=True,
-        null=True,
-        help_text="Original file key if the Markdown from another file",
-    )

src/backend/chat/serializers.py

@@ -1,201 +1,16 @@
 """Serializers for chat application."""
 
-from typing import Optional
-from urllib.parse import quote
-
-from django.conf import settings
-
-from django_pydantic_field.rest_framework import SchemaField  # pylint: disable=no-name-in-module
-from drf_spectacular.utils import extend_schema_field
 from rest_framework import serializers
 
-from core.file_upload.enums import AttachmentStatus
-from core.file_upload.utils import generate_upload_policy
-
 from chat import models
-from chat.ai_sdk_types import UIMessage
 
 
 class ChatConversationSerializer(serializers.ModelSerializer):
     """Serializer for chat conversations."""
 
     owner = serializers.HiddenField(default=serializers.CurrentUserDefault())
-    messages = SchemaField(schema=list[UIMessage], read_only=True)
 
     class Meta:  # pylint: disable=missing-class-docstring
         model = models.ChatConversation
         fields = ["id", "title", "created_at", "updated_at", "messages", "owner"]
         read_only_fields = ["id", "created_at", "updated_at", "messages"]
-
-
-class ChatConversationInputSerializer(serializers.Serializer):
-    """
-    Used to serialize input from Vercel AI SDK when using conversation endpoint.
-
-    See ChatViewSet().post_conversation(...) method for more details.
-    """
-
-    messages = SchemaField(schema=list[UIMessage])
-
-    def update(self, instance, validated_data):
-        """Update method is not applicable in this context."""
-        raise NotImplementedError("`update()` should not be used in this context.")
-
-    def create(self, validated_data):
-        """Create method is not applicable in this context."""
-        raise NotImplementedError("`create()` should not be used in this context.")
-
-    def validate_messages(self, messages):
-        """Validate that messages is not empty."""
-        if not messages:
-            raise serializers.ValidationError("This list must not be empty.")
-        return messages
-
-
-class ChatConversationRequestSerializer(serializers.Serializer):
-    """
-    Used to serialize query parameters.
-
-    See ChatViewSet().post_conversation(...) method for more details.
-    """
-
-    protocol = serializers.CharField(
-        required=False,
-        default="data",
-        help_text="Protocol version to use for the conversation (text or data).",
-        allow_blank=True,
-        trim_whitespace=True,
-    )
-
-    force_web_search = serializers.BooleanField(
-        required=False,
-        default=False,
-        help_text="Force web search.",
-    )
-    model_hrid = serializers.CharField(
-        required=False,
-        default=None,
-        help_text="HRID of the model to use for the conversation.",
-        allow_blank=True,
-        trim_whitespace=True,
-    )
-
-    def update(self, instance, validated_data):
-        """Update method is not applicable in this context."""
-        raise NotImplementedError("`update()` should not be used in this context.")
-
-    def create(self, validated_data):
-        """Create method is not applicable in this context."""
-        raise NotImplementedError("`create()` should not be used in this context.")
-
-    def validate_protocol(self, value):
-        """Validate the protocol field."""
-        if value not in ["text", "data"]:
-            raise serializers.ValidationError("Protocol must be either 'text' or 'data'.")
-        return value
-
-    def validate_model_hrid(self, value):
-        """Validate the model_hrid field."""
-        value = value or None  # Convert empty string to None
-
-        if value and value not in settings.LLM_CONFIGURATIONS:
-            raise serializers.ValidationError("Invalid model_hrid.")
-
-        return value
-
-
-class LLModelSerializer(serializers.Serializer):  # pylint: disable=abstract-method
-    """Serializer for LL models."""
-
-    hrid = serializers.CharField(help_text="Human-readable ID of the model.")
-    model_name = serializers.CharField(help_text="Name of the model.")
-    human_readable_name = serializers.CharField(help_text="Human-readable name of the model.")
-    icon = serializers.CharField(
-        help_text="Icon representing the model.",
-        allow_blank=True,
-        required=False,
-    )
-    is_active = serializers.BooleanField(
-        help_text="Indicates if the model is active and available for selection.",
-        required=False,
-        default=True,
-    )
-
-    # Computed field to indicate if the model is the default model
-    is_default = serializers.SerializerMethodField(
-        help_text="Indicates if the model is the default model.",
-    )
-
-    @staticmethod
-    @extend_schema_field(serializers.BooleanField)
-    def get_is_default(obj) -> bool:
-        """Check if the model is the default model."""
-        return obj.hrid == settings.LLM_DEFAULT_MODEL_HRID
-
-
-class ChatMessageCategoricalScoreSerializer(serializers.Serializer):  # pylint: disable=abstract-method
-    """Serializer for chat message scores."""
-
-    message_id = serializers.CharField(help_text="ID of the message to score.")
-    name = serializers.HiddenField(default="sentiment")
-    value = serializers.ChoiceField(
-        choices=["positive", "negative"],
-        help_text="Sentiment of the score.",
-    )
-
-
-class LLMConfigurationSerializer(serializers.Serializer):  # pylint: disable=abstract-method
-    """Serializer for LLM configuration."""
-
-    models = LLModelSerializer(many=True)
-
-
-class ChatConversationAttachmentSerializer(serializers.ModelSerializer):
-    """Serializer for chat conversation attachments."""
-
-    url = serializers.SerializerMethodField()
-
-    class Meta:  # pylint: disable=missing-class-docstring
-        model = models.ChatConversationAttachment
-        fields = ["id", "key", "content_type", "file_name", "size", "upload_state", "url"]
-        read_only_fields = ["id", "key", "content_type", "file_name", "size", "upload_state"]
-
-    def get_url(self, attachment) -> str | None:
-        """Return the URL of the attachment."""
-        if attachment.upload_state not in (
-            AttachmentStatus.FILE_TOO_LARGE_TO_ANALYZE,
-            AttachmentStatus.SUSPICIOUS,
-            AttachmentStatus.READY,
-        ):
-            return None
-
-        return f"{settings.MEDIA_BASE_URL}{settings.MEDIA_URL}{quote(attachment.key)}"
-
-
-class CreateChatConversationAttachmentSerializer(serializers.ModelSerializer):
-    """Serializer for creating chat conversation attachments."""
-
-    policy = serializers.SerializerMethodField()
-    uploaded_by = serializers.HiddenField(default=serializers.CurrentUserDefault())
-    key = serializers.CharField(read_only=True)  # Key is generated server-side
-
-    class Meta:  # pylint: disable=missing-class-docstring
-        model = models.ChatConversationAttachment
-        fields = ["id", "key", "content_type", "file_name", "size", "policy", "uploaded_by"]
-
-    def get_policy(self, attachment) -> str:
-        """Return the policy to use if the item is a file."""
-        return generate_upload_policy(attachment.key)
-
-    def validate_size(self, size: Optional[int]) -> Optional[int]:
-        """Validate that the size is not greater than the maximum allowed size."""
-        if not size:
-            return size
-
-        if size > settings.ATTACHMENT_MAX_SIZE:
-            max_size = settings.ATTACHMENT_MAX_SIZE // (1024 * 1024)
-            raise serializers.ValidationError(
-                f"File size exceeds the maximum limit of {max_size:d} MB."
-            )
-
-        return size

src/backend/chat/tests/__init__.py

@@ -1 +0,0 @@
-"""Tests for chat application."""

src/backend/chat/tests/agent_rag/document_converter/test_markitdown.py

@@ -1,32 +0,0 @@
-"""
-Unit tests for the DocumentConverter.
-
-Only for coverage as the DocumentConverter is a simple wrapper around MarkItDown.
-"""
-
-from io import BytesIO
-from unittest.mock import MagicMock, patch
-
-from chat.agent_rag.document_converter.markitdown import DocumentConverter
-
-
-@patch("chat.agent_rag.document_converter.markitdown.MarkItDown")
-def test_document_converter(mock_markitdown: MagicMock):
-    """Test that the DocumentConverter calls the underlying MarkItDown converter."""
-    mock_conversion = MagicMock()
-    mock_conversion.text_content = "converted text"
-    mock_markitdown.return_value.convert_stream.return_value = mock_conversion
-
-    converter = DocumentConverter()
-
-    result = converter.convert_raw(
-        name="test.pdf",
-        content_type="application/pdf",
-        content=b"test content",
-    )
-
-    assert result == "converted text"
-    converter.converter.convert_stream.assert_called_once()  # pylint: disable=no-member
-    args, kwargs = converter.converter.convert_stream.call_args  # pylint: disable=no-member
-    assert isinstance(args[0], BytesIO)
-    assert kwargs["file_extension"] == ".pdf"

src/backend/chat/tests/agent_rag/document_search/test_albert_api.py

@@ -1,243 +0,0 @@
-"""Tests for the Albert RAG document search API."""
-# pylint: disable=protected-access
-
-from io import BytesIO
-from unittest.mock import patch
-
-import pytest
-import responses
-from requests import HTTPError
-
-from chat.agent_rag.document_search.albert_api import AlbertRagDocumentSearch
-from chat.factories import ChatConversationFactory
-
-pytestmark = pytest.mark.django_db()
-
-
-def test_albert_collection_id_property():
-    """Test the _albert_collection_id property."""
-    conversation = ChatConversationFactory()
-    assert (
-        AlbertRagDocumentSearch(conversation)._albert_collection_id
-        == f"conversation-{conversation.pk}"
-    )
-
-
-def test_collection_id_property():
-    """Test the collection_id property."""
-    conversation = ChatConversationFactory()
-    search = AlbertRagDocumentSearch(conversation)
-
-    # When collection_id is None
-    conversation.collection_id = None
-    assert search.collection_id is None
-
-    # When collection_id is set
-    conversation.collection_id = "123"
-    assert search.collection_id == 123
-
-
-@responses.activate
-def test_create_collection_success():
-    """Test _create_collection successfully creates a collection."""
-    conversation = ChatConversationFactory()
-    search = AlbertRagDocumentSearch(conversation)
-    responses.add(
-        responses.POST,
-        search._collections_endpoint,
-        json={"id": "456"},
-        status=201,
-    )
-
-    assert search._create_collection() is True
-    assert conversation.collection_id == "456"
-
-
-@responses.activate
-def test_create_collection_failure():
-    """Test _create_collection handles API errors."""
-    conversation = ChatConversationFactory()
-    search = AlbertRagDocumentSearch(conversation)
-    responses.add(
-        responses.POST,
-        search._collections_endpoint,
-        status=500,
-    )
-
-    with pytest.raises(HTTPError):
-        search._create_collection()
-
-
-@responses.activate
-def test_parse_pdf_document_success():
-    """Test _parse_pdf_document successfully parses a PDF."""
-    conversation = ChatConversationFactory()
-    search = AlbertRagDocumentSearch(conversation)
-    responses.add(
-        responses.POST,
-        search._pdf_parser_endpoint,
-        json={"data": [{"content": "Page 1"}, {"content": "Page 2"}]},
-        status=200,
-    )
-
-    content = search._parse_pdf_document("test.pdf", "application/pdf", BytesIO(b"pdf_content"))
-    assert content == "Page 1\n\nPage 2"
-
-
-@responses.activate
-def test_parse_pdf_document_failure():
-    """Test _parse_pdf_document handles API errors."""
-    conversation = ChatConversationFactory()
-    search = AlbertRagDocumentSearch(conversation)
-    responses.add(
-        responses.POST,
-        search._pdf_parser_endpoint,
-        status=500,
-    )
-
-    with pytest.raises(HTTPError):
-        search._parse_pdf_document("test.pdf", "application/pdf", BytesIO(b"pdf_content"))
-
-
-@patch("chat.agent_rag.document_search.albert_api.AlbertRagDocumentSearch._parse_pdf_document")
-def test_parse_document_pdf(mock_parse_pdf):
-    """Test parse_document for PDF content."""
-    conversation = ChatConversationFactory()
-    search = AlbertRagDocumentSearch(conversation)
-    mock_parse_pdf.return_value = "Parsed PDF content"
-    result = search.parse_document("test.pdf", "application/pdf", BytesIO(b"pdf"))
-    assert result == "Parsed PDF content"
-    mock_parse_pdf.assert_called_once()
-
-
-@patch("chat.agent_rag.document_search.albert_api.DocumentConverter")
-def test_parse_document_other(mock_converter):
-    """Test parse_document for other content types."""
-    conversation = ChatConversationFactory()
-    search = AlbertRagDocumentSearch(conversation)
-    mock_converter.return_value.convert_raw.return_value = "Converted content"
-    result = search.parse_document("test.txt", "text/plain", BytesIO(b"text"))
-    assert result == "Converted content"
-    mock_converter.return_value.convert_raw.assert_called_once()
-
-
-@responses.activate
-def test_store_document_success():
-    """Test _store_document successfully stores a document."""
-    conversation = ChatConversationFactory(collection_id="123")
-    search = AlbertRagDocumentSearch(conversation)
-    responses.add(
-        responses.POST,
-        search._documents_endpoint,
-        json={"id": "doc1"},
-        status=201,
-    )
-
-    search._store_document("test_doc", "some content")
-    assert len(responses.calls) == 1
-    assert responses.calls[0].request.url == search._documents_endpoint
-
-
-@responses.activate
-@patch("chat.agent_rag.document_search.albert_api.AlbertRagDocumentSearch._create_collection")
-def test_store_document_creates_collection(mock_create_collection):
-    """Test _store_document creates a collection if one doesn't exist."""
-    conversation = ChatConversationFactory(collection_id=None)
-    search = AlbertRagDocumentSearch(conversation)
-
-    def set_collection_id(*args, **kwargs):
-        conversation.collection_id = "123"
-        return True
-
-    mock_create_collection.side_effect = set_collection_id
-
-    responses.add(
-        responses.POST,
-        search._documents_endpoint,
-        json={"id": "doc1"},
-        status=201,
-    )
-
-    search._store_document("test_doc", "some content")
-    mock_create_collection.assert_called_once()
-    assert conversation.collection_id == "123"
-
-
-@patch(
-    "chat.agent_rag.document_search.albert_api.AlbertRagDocumentSearch._create_collection",
-    return_value=False,
-)
-def test_store_document_create_collection_fails(mock_create_collection):
-    """Test _store_document raises error if collection creation fails."""
-    conversation = ChatConversationFactory(collection_id=None)
-    search = AlbertRagDocumentSearch(conversation)
-    with pytest.raises(RuntimeError, match="Failed to create or retrieve the collection."):
-        search._store_document("test_doc", "some content")
-    mock_create_collection.assert_called_once()
-
-
-@patch("chat.agent_rag.document_search.albert_api.AlbertRagDocumentSearch.parse_document")
-@patch("chat.agent_rag.document_search.albert_api.AlbertRagDocumentSearch._store_document")
-def test_parse_and_store_document(mock_store, mock_parse):
-    """Test parse_and_store_document orchestrates parsing and storing."""
-    conversation = ChatConversationFactory()
-    search = AlbertRagDocumentSearch(conversation)
-    mock_parse.return_value = "parsed content"
-    name = "test.txt"
-    content_type = "text/plain"
-    content = BytesIO(b"text")
-
-    result = search.parse_and_store_document(name, content_type, content)
-
-    assert result == "parsed content"
-    mock_parse.assert_called_once_with(name, content_type, content)
-    mock_store.assert_called_once_with(name, "parsed content")
-
-
-@responses.activate
-def test_search_success():
-    """Test search successfully returns results."""
-    conversation = ChatConversationFactory(collection_id="123")
-    search = AlbertRagDocumentSearch(conversation)
-    mock_response = {
-        "data": [
-            {
-                "method": "semantic",
-                "chunk": {
-                    "id": 1,
-                    "content": "Relevant content snippet.",
-                    "metadata": {"document_name": "doc1.txt"},
-                },
-                "score": 0.9,
-            }
-        ],
-        "usage": {"prompt_tokens": 10, "completion_tokens": 20},
-    }
-    responses.post(
-        url=search._search_endpoint,
-        json=mock_response,
-        status=200,
-    )
-
-    results = search.search("test query")
-
-    assert len(results.data) == 1
-    assert results.data[0].content == "Relevant content snippet."
-    assert results.data[0].url == "doc1.txt"
-    assert results.data[0].score == 0.9
-    assert results.usage.prompt_tokens == 10
-    assert results.usage.completion_tokens == 20
-
-
-@responses.activate
-def test_search_failure():
-    """Test search handles API errors."""
-    conversation = ChatConversationFactory(collection_id="123")
-    search = AlbertRagDocumentSearch(conversation)
-    responses.post(
-        url=search._search_endpoint,
-        status=500,
-    )
-
-    with pytest.raises(HTTPError):
-        search.search("test query")

src/backend/chat/tests/agent_rag/test_albert_api_constants.py

@@ -1,60 +0,0 @@
-"""Test suite for albert_api_constants.py."""
-
-import pytest
-from pydantic import ValidationError
-
-from chat.agent_rag.albert_api_constants import (
-    SearchArgs,
-    SearchMethod,
-    SearchRequest,
-)
-
-
-def test_search_request_model_dump_json():
-    """Test that SearchRequest.model_dump(mode='json') works correctly."""
-    search_request = SearchRequest(prompt="test prompt")
-    assert search_request.model_dump(mode="json") == {
-        "collections": [],
-        "k": 4,
-        "method": "semantic",
-        "prompt": "test prompt",
-        "rff_k": 20,
-        "score_threshold": 0.0,
-        "web_search": False,
-        "web_search_k": 5,
-    }
-
-
-def test_search_args_score_threshold_valid():
-    """Test that score_threshold is valid for semantic and multiagent search methods."""
-    try:
-        SearchArgs(method=SearchMethod.SEMANTIC, score_threshold=0.5)
-        SearchArgs(method=SearchMethod.MULTIAGENT, score_threshold=0.5)
-    except ValidationError:
-        pytest.fail("ValidationError was raised unexpectedly for valid search methods.")
-
-
-def test_search_args_score_threshold_invalid():
-    """Test that score_threshold raises ValueError for hybrid and lexical search methods."""
-    with pytest.raises(ValidationError) as excinfo:
-        SearchArgs(method=SearchMethod.HYBRID, score_threshold=0.5)
-    assert "Score threshold is only available for semantic and multiagent search methods." in str(
-        excinfo.value
-    )
-
-    with pytest.raises(ValidationError) as excinfo:
-        SearchArgs(method=SearchMethod.LEXICAL, score_threshold=0.5)
-    assert "Score threshold is only available for semantic and multiagent search methods." in str(
-        excinfo.value
-    )
-
-
-def test_search_args_no_score_threshold():
-    """Test that no error is raised when score_threshold is not set."""
-    try:
-        SearchArgs(method=SearchMethod.HYBRID)
-        SearchArgs(method=SearchMethod.LEXICAL)
-        SearchArgs(method=SearchMethod.SEMANTIC)
-        SearchArgs(method=SearchMethod.MULTIAGENT)
-    except ValidationError:
-        pytest.fail("ValidationError was raised unexpectedly when score_threshold is not set.")

src/backend/chat/tests/agent_rag/web_search/test_albert_api.py

@@ -1,105 +0,0 @@
-"""Unit tests for the Albert API web search manager."""
-
-import json
-
-import pytest
-import requests
-import responses
-
-from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
-from chat.agent_rag.web_search.albert_api import AlbertWebSearchManager
-
-
-@pytest.fixture(autouse=True)
-def albert_api_settings(settings):
-    """Fixture to set Albert API settings for tests."""
-    settings.ALBERT_API_URL = "http://test-albert-api.com"
-    settings.ALBERT_API_KEY = "test-key"
-
-
-@pytest.mark.parametrize(
-    "url, expected",
-    [
-        ("http://example.com/page.html", "http://example.com/page"),
-        ("http://example.com/page", "http://example.com/page"),
-        ("http://example.com/.html", "http://example.com/"),
-    ],
-)
-def test_clean_url(url, expected):
-    """Test the _clean_url static method."""
-    assert AlbertWebSearchManager._clean_url(url) == expected  # pylint: disable=protected-access
-
-
-@responses.activate
-def test_web_search_success(settings):
-    """Test a successful web search."""
-    settings.RAG_WEB_SEARCH_MAX_RESULTS = 20
-    settings.RAG_WEB_SEARCH_CHUNK_NUMBER = 10
-
-    mock_albert_api = responses.post(
-        "http://test-albert-api.com/v1/search",
-        json={
-            "data": [
-                {
-                    "method": "semantic",
-                    "chunk": {
-                        "id": 123,
-                        "content": "This is a test chunk.",
-                        "metadata": {
-                            "document_name": "http://example.com/test.html",
-                            "document_type": "html",
-                        },
-                    },
-                    "score": 0.9,
-                }
-            ],
-            "usage": {"prompt_tokens": 10, "completion_tokens": 20},
-        },
-        status=200,
-        content_type="application/json",
-    )
-
-    results = AlbertWebSearchManager().web_search("test query")
-    assert results == RAGWebResults(
-        data=[
-            RAGWebResult(url="http://example.com/test", content="This is a test chunk.", score=0.9)
-        ],
-        usage=RAGWebUsage(prompt_tokens=10, completion_tokens=20),
-    )
-
-    # Verify the request payload
-    request = mock_albert_api.calls[0].request
-    assert json.loads(request.body) == {
-        "prompt": "test query",
-        "web_search": True,
-        "web_search_k": 20,  # Default value from settings
-        "k": 10,  # Default value from settings
-    }
-
-
-def test_web_search_empty_query():
-    """Test web_search with an empty query."""
-    with pytest.raises(ValueError, match="Search query cannot be empty."):
-        AlbertWebSearchManager().web_search("   ")
-
-
-@responses.activate
-def test_web_search_http_error():
-    """Test web_search with an HTTP error from the API."""
-    responses.post("http://test-albert-api.com/v1/search", status=500)
-    with pytest.raises(requests.HTTPError):
-        AlbertWebSearchManager().web_search("test query")
-
-
-@responses.activate
-def test_web_search_json_decode_error():
-    """Test web_search with a JSON decode error from the API."""
-    responses.post(
-        "http://test-albert-api.com/v1/search",
-        body="invalid json",
-        status=200,
-        content_type="application/json",
-    )
-
-    with pytest.raises(requests.exceptions.JSONDecodeError):
-        AlbertWebSearchManager().web_search("test query")

src/backend/chat/tests/agents/test_base_agent.py

@@ -1,112 +0,0 @@
-"""Tests for the BaseAgent class and its model initialization logic."""
-# pylint: disable=protected-access
-
-from pydantic_ai.models.mistral import MistralModel
-from pydantic_ai.models.openai import OpenAIChatModel
-
-from chat.agents.base import BaseAgent
-from chat.llm_configuration import LLModel, LLMProfile, LLMProvider
-
-
-def test_not_custom_model(monkeypatch, settings):
-    """Test that a model without a provider relies on Pydantic AI detection."""
-    settings.LLM_CONFIGURATIONS = {
-        "gpt-4": LLModel(
-            hrid="gpt-4",
-            model_name="openai:gpt-4",
-            human_readable_name="GPT-4",
-            is_active=True,
-            system_prompt="direct",
-            tools=[],
-        ),
-    }
-
-    # Required for OpenAI models client initialization
-    monkeypatch.setenv("OPENAI_API_KEY", "hello")
-
-    agent = BaseAgent(model_hrid="gpt-4")
-    assert isinstance(agent._model, OpenAIChatModel)
-
-
-def test_custom_model_openai(settings):
-    """Test that a custom OpenAI model is initialized correctly."""
-    settings.LLM_CONFIGURATIONS = {
-        "openai-compatible-model": LLModel(
-            hrid="custom-gpt-4",
-            model_name="gpt-4",
-            human_readable_name="Custom GPT-4",
-            profile=None,
-            provider=LLMProvider(
-                hrid="openai",
-                kind="openai",
-                base_url="https://test.vllm/v1",
-                api_key="testkey",
-            ),
-            is_active=True,
-            system_prompt="direct",
-            tools=[],
-        ),
-    }
-
-    agent = BaseAgent(model_hrid="openai-compatible-model")
-    assert isinstance(agent._model, OpenAIChatModel)
-
-
-def test_custom_model_mistral(settings):
-    """Test that a custom Mistral model is initialized correctly."""
-    settings.LLM_CONFIGURATIONS = {
-        "mistral-model": LLModel(
-            hrid="mistral-model",
-            model_name="mistral-7b-instruct-v0.1",
-            human_readable_name="Mistral 7B Instruct",
-            profile=None,
-            provider=LLMProvider(
-                hrid="mistral",
-                kind="mistral",
-                base_url="https://api.mistral.ai/v1",
-                api_key="testkey",
-            ),
-            is_active=True,
-            system_prompt="direct",
-            tools=[],
-        ),
-    }
-
-    agent = BaseAgent(model_hrid="mistral-model")
-
-    assert isinstance(agent._model, MistralModel)
-
-    import pydantic_ai.models.mistral as mistral_models  # noqa: PLC0415 # pylint: disable=import-outside-toplevel
-
-    assert mistral_models.__safe_map_patched__ is True  # pylint: disable=protected-access
-
-
-def test_custom_model_openai_profile(settings):
-    """Test that a custom OpenAI model with profile is initialized correctly."""
-    settings.LLM_CONFIGURATIONS = {
-        "openai-model": LLModel(
-            hrid="openai-model",
-            model_name="some-openai-model",
-            human_readable_name="Some OpenAI Model",
-            profile=LLMProfile(
-                supports_json_schema_output=False,
-                openai_supports_strict_tool_definition=False,
-            ),
-            provider=LLMProvider(
-                hrid="openai",
-                kind="openai",
-                base_url="https://test.vllm/v1",
-                api_key="testkey",
-            ),
-            is_active=True,
-            system_prompt="direct",
-            tools=[],
-        ),
-    }
-
-    agent = BaseAgent(model_hrid="openai-model")
-
-    assert isinstance(agent._model, OpenAIChatModel)
-    assert agent._model.profile.supports_tools is True
-    assert agent._model.profile.supports_json_schema_output is False
-    assert agent._model.profile.supports_json_object_output is True

src/backend/chat/tests/agents/test_build_conversation_agent.py

@@ -1,119 +0,0 @@
-"""Test cases for the ConversationAgent class in the chat.clients.pydantic_ai module."""
-
-# pylint:disable=protected-access
-
-import pytest
-import responses
-from freezegun import freeze_time
-from pydantic_ai import Agent
-from pydantic_ai.models.openai import OpenAIChatModel
-from pydantic_ai.models.test import TestModel
-
-from chat.agents.conversation import ConversationAgent
-from chat.clients.pydantic_ai import ContextDeps
-
-
-@pytest.fixture(autouse=True)
-def base_settings(settings):
-    """Set up base settings for the tests."""
-    settings.AI_BASE_URL = "https://api.llm.com/v1/"
-    settings.AI_API_KEY = "test-key"
-    settings.AI_MODEL = "model-123"
-    settings.AI_AGENT_INSTRUCTIONS = "You are a helpful assistant"
-    settings.AI_AGENT_TOOLS = []
-
-
-def test_build_pydantic_agent_success_no_tools():
-    """Test successful agent creation without tools."""
-    agent = ConversationAgent(model_hrid="default-model")
-    assert isinstance(agent, Agent)
-
-    assert agent._system_prompts == ("You are a helpful assistant",)
-    assert agent._instructions == []
-    assert isinstance(agent.model, OpenAIChatModel)
-    assert agent.model.model_name == "model-123"
-    assert str(agent.model.client.base_url) == "https://api.llm.com/v1/"
-    assert agent.model.client.api_key == "test-key"
-    assert agent._function_toolset.tools == {}
-
-
-def test_build_pydantic_agent_with_tools(settings):
-    """Test successful agent creation with tools."""
-    settings.AI_AGENT_TOOLS = ["get_current_weather"]
-
-    agent = ConversationAgent(model_hrid="default-model")
-    assert isinstance(agent, Agent)
-
-    assert agent._system_prompts == ("You are a helpful assistant",)
-    assert agent._instructions == []
-    assert isinstance(agent.model, OpenAIChatModel)
-    assert agent.model.model_name == "model-123"
-    assert str(agent.model.client.base_url) == "https://api.llm.com/v1/"
-    assert agent.model.client.api_key == "test-key"
-    assert list(agent._function_toolset.tools.keys()) == ["get_current_weather"]
-
-
-@freeze_time("2025-07-25T10:36:35.297675Z")
-def test_add_dynamic_system_prompt():
-    """
-    Ensure add_the_date and enforce_response_language system prompt are registered
-    and returns proper values.
-    """
-    agent = ConversationAgent(model_hrid="default-model")
-
-    assert len(agent._system_prompt_functions) == 2
-
-    assert agent._system_prompt_functions[0].function.__name__ == "add_the_date"
-    assert agent._system_prompt_functions[0].function() == "Today is Friday 25/07/2025."
-
-    assert agent._system_prompt_functions[1].function.__name__ == "enforce_response_language"
-    assert agent._system_prompt_functions[1].function() == ""
-
-    agent = ConversationAgent(model_hrid="default-model", language="fr-fr")
-    assert agent._system_prompt_functions[1].function() == "Answer in french."
-
-
-def test_agent_get_web_search_tool_name(settings):
-    """Test the web_search_available method."""
-    settings.AI_AGENT_TOOLS = ["get_current_weather", "web_search_albert_rag"]
-    agent = ConversationAgent(model_hrid="default-model")
-    assert agent.get_web_search_tool_name() == "web_search_albert_rag"
-
-    settings.AI_AGENT_TOOLS = ["get_current_weather"]
-    agent = ConversationAgent(model_hrid="default-model")
-    assert agent.get_web_search_tool_name() is None
-
-    settings.AI_AGENT_TOOLS = ["get_current_weather", "web_search_tavily", "web_search_albert_rag"]
-    agent = ConversationAgent(model_hrid="default-model")
-    assert agent.get_web_search_tool_name() == "web_search_tavily"
-
-
-@responses.activate
-def test_web_search_tool_avalability(settings):
-    """Test the web search tool availability according to context."""
-    responses.add(
-        responses.POST,
-        "https://api.tavily.com/search",
-        json={"results": []},
-        status=200,
-    )
-    context_deps = ContextDeps(conversation=None, user=None, web_search_enabled=True)
-
-    # No tools (context allows web search, but no tool configured)
-    agent = ConversationAgent(model_hrid="default-model")
-    with agent.override(model=TestModel(), deps=context_deps):
-        response = agent.run_sync("What tools do you have?")
-        assert response.output == "success (no tool calls)"
-
-    # Tool configured, context allows web search
-    settings.AI_AGENT_TOOLS = ["web_search_tavily"]
-    agent = ConversationAgent(model_hrid="default-model")  # re-init to pick up new settings
-    with agent.override(model=TestModel(), deps=context_deps):
-        response = agent.run_sync("What tools do you have?")
-        assert response.output == '{"web_search_tavily":[]}'
-
-    # Tool configured, context disables web search
-    context_deps.web_search_enabled = False
-    with agent.override(model=TestModel(), deps=context_deps):
-        response = agent.run_sync("What tools do you have?")
-        assert response.output == "success (no tool calls)"

src/backend/chat/tests/clients/pydantic_ai/test_stream_methods.py

@@ -1,109 +0,0 @@
-"""Unit tests for AIAgentService stream methods."""
-
-from unittest.mock import patch
-
-import pytest
-from asgiref.sync import sync_to_async
-
-from chat.ai_sdk_types import UIMessage
-from chat.clients.pydantic_ai import AIAgentService
-from chat.factories import ChatConversationFactory
-from chat.vercel_ai_sdk.core import events_v4
-
-pytestmark = pytest.mark.django_db()
-
-
-@pytest.fixture(autouse=True)
-def base_settings(settings):
-    """Set up base settings for the tests."""
-    settings.AI_BASE_URL = "https://api.llm.com/v1/"
-    settings.AI_API_KEY = "test-key"
-    settings.AI_MODEL = "model-123"
-    settings.AI_AGENT_INSTRUCTIONS = "You are a helpful assistant"
-    settings.AI_AGENT_TOOLS = []
-
-
-@pytest.fixture(name="ui_messages")
-def ui_messages_fixture():
-    """Fixture for test UI messages."""
-    return [UIMessage(id="msg-1", role="user", content="Hello, how are you?", parts=[])]
-
-
-@patch("chat.clients.pydantic_ai.convert_async_generator_to_sync")
-def test_stream_text_delegates_to_async(mock_convert, ui_messages):
-    """Test stream_text method delegates to async version."""
-    conversation = ChatConversationFactory()
-    service = AIAgentService(conversation, user=conversation.owner)
-    mock_convert.return_value = iter(["Hello", " world"])
-
-    result = service.stream_text(ui_messages, force_web_search=True)
-
-    mock_convert.assert_called_once()
-    assert result == mock_convert.return_value
-
-
-@patch("chat.clients.pydantic_ai.convert_async_generator_to_sync")
-def test_stream_data_delegates_to_async(mock_convert, ui_messages):
-    """Test stream_data method delegates to async version."""
-    conversation = ChatConversationFactory()
-    service = AIAgentService(conversation, user=conversation.owner)
-    mock_convert.return_value = iter(['0:"Hello"\n', 'd:{"finishReason":"stop"}\n'])
-
-    result = service.stream_data(ui_messages, force_web_search=False)
-
-    mock_convert.assert_called_once()
-    assert result == mock_convert.return_value
-
-
-@pytest.mark.asyncio
-async def test_stream_text_async_filters_text_deltas(ui_messages):
-    """Test stream_text_async only yields text deltas."""
-    conversation = await sync_to_async(ChatConversationFactory)()
-    service = AIAgentService(conversation, user=conversation.owner)
-
-    # Mock _run_agent to return various delta types
-    async def mock_run_agent(*args, **kwargs):
-        yield events_v4.TextPart(text="Hello")
-        yield events_v4.ToolCallStreamingStartPart(tool_call_id="123", tool_name="search")
-        yield events_v4.TextPart(text=" world")
-        yield events_v4.FinishMessagePart(
-            finish_reason=events_v4.FinishReason.STOP,
-            usage=events_v4.Usage(
-                prompt_tokens=120,
-                completion_tokens=456,
-            ),
-        )
-
-    with patch.object(service, "_run_agent", side_effect=mock_run_agent):
-        results = []
-        async for result in service.stream_text_async(ui_messages):
-            results.append(result)
-
-        assert results == ["Hello", " world"]
-
-
-@pytest.mark.asyncio
-async def test_stream_data_async_formats_as_sdk_events(ui_messages):
-    """Test stream_data_async formats events correctly."""
-    conversation = await sync_to_async(ChatConversationFactory)()
-    service = AIAgentService(conversation, user=conversation.owner)
-
-    async def mock_run_agent(*args, **kwargs):
-        yield events_v4.TextPart(text="Hello")
-        yield events_v4.FinishMessagePart(
-            finish_reason=events_v4.FinishReason.STOP,
-            usage=events_v4.Usage(
-                prompt_tokens=120,
-                completion_tokens=456,
-            ),
-        )
-
-    with patch.object(service, "_run_agent", side_effect=mock_run_agent):
-        results = []
-        async for result in service.stream_data_async(ui_messages):
-            results.append(result)
-
-        assert results == [
-            '0:"Hello"\n',
-            'd:{"finishReason":"stop","usage":{"promptTokens":120,"completionTokens":456}}\n',
-        ]

src/backend/chat/tests/clients/pydantic_ui_message_converter/test_model_message_to_ui_message.py

@@ -1,331 +0,0 @@
-"""Tests for converting ModelMessage to UIMessage using Pydantic AI types."""
-
-import datetime
-import json
-import uuid
-from unittest.mock import patch
-
-from django.utils import timezone
-
-import pytest
-from freezegun import freeze_time
-from pydantic_ai import ImageUrl
-from pydantic_ai.messages import (
-    AudioUrl,
-    BinaryContent,
-    DocumentUrl,
-    ModelRequest,
-    ModelResponse,
-    RetryPromptPart,
-    SystemPromptPart,
-    TextPart,
-    ThinkingPart,
-    ToolCallPart,
-    UserPromptPart,
-    VideoUrl,
-)
-
-from chat.ai_sdk_types import (
-    Attachment,
-    ReasoningDetailText,
-    ReasoningUIPart,
-    TextUIPart,
-    ToolInvocationCall,
-    ToolInvocationUIPart,
-    UIMessage,
-)
-from chat.clients.pydantic_ui_message_converter import model_message_to_ui_message
-
-
-@pytest.fixture(autouse=True)
-def mock_uuid4_fixture():
-    """Fixture to mock UUID generation for testing."""
-    with patch("uuid.uuid4", return_value=uuid.UUID("f0cc3bb5-f207-401b-8281-4cba6202991d")):
-        yield
-
-
-def test_model_message_to_ui_message_text_user_full():
-    """Test converting a ModelRequest with UserPromptPart containing text to UIMessage."""
-    timestamp = datetime.datetime.now()
-    model_message = ModelRequest(
-        parts=[UserPromptPart(content="Hello!", timestamp=timestamp)], kind="request"
-    )
-    expected = UIMessage(
-        id="f0cc3bb5-f207-401b-8281-4cba6202991d",  # Mocked UUID
-        role="user",
-        content="Hello!",
-        parts=[TextUIPart(type="text", text="Hello!")],
-        createdAt=timestamp,
-    )
-    result = model_message_to_ui_message(model_message)
-    assert result == expected
-
-
-@freeze_time()
-def test_model_message_to_ui_message_text_assistant_full():
-    """Test converting a ModelResponse with TextPart to UIMessage."""
-    model_message = ModelResponse(parts=[TextPart(content="Hi there!")])
-    expected = UIMessage(
-        id="f0cc3bb5-f207-401b-8281-4cba6202991d",  # Mocked UUID
-        role="assistant",
-        content="Hi there!",
-        parts=[TextUIPart(type="text", text="Hi there!")],
-        createdAt=timezone.now(),
-    )
-    result = model_message_to_ui_message(model_message)
-    assert result == expected
-
-
-@freeze_time()
-def test_model_message_to_ui_message_tool_call_full():
-    """Test converting a ModelResponse with ToolCallPart to UIMessage."""
-    args = {"foo": "bar"}
-    model_message = ModelResponse(
-        parts=[ToolCallPart(tool_call_id="id1", tool_name="tool", args=args)]
-    )
-    expected = UIMessage(
-        id="f0cc3bb5-f207-401b-8281-4cba6202991d",  # Mocked UUID
-        role="assistant",
-        content="",
-        parts=[
-            ToolInvocationUIPart(
-                type="tool-invocation",
-                toolInvocation=ToolInvocationCall(
-                    state="call",
-                    toolCallId="id1",
-                    toolName="tool",
-                    args=args,
-                ),
-            )
-        ],
-        createdAt=timezone.now(),
-    )
-    result = model_message_to_ui_message(model_message)
-    assert result == expected
-
-
-@freeze_time()
-def test_model_message_to_ui_message_reasoning_full():
-    """Test converting a ModelResponse with ThinkingPart to UIMessage."""
-    model_message = ModelResponse(parts=[ThinkingPart(content="reason", signature="sig")])
-    expected = UIMessage(
-        id="f0cc3bb5-f207-401b-8281-4cba6202991d",  # Mocked UUID
-        role="assistant",
-        content="",
-        parts=[
-            ReasoningUIPart(
-                type="reasoning",
-                reasoning="reason",
-                details=[ReasoningDetailText(type="text", text="reason", signature="sig")],
-            )
-        ],
-        createdAt=timezone.now(),
-    )
-    result = model_message_to_ui_message(model_message)
-    assert result.id == expected.id
-    assert result.role == expected.role
-    assert result.content == expected.content
-    assert result.createdAt == expected.createdAt
-    assert len(result.parts) == 1
-    parts_list = list(result.parts)
-    part = parts_list[0]
-    assert isinstance(part, ReasoningUIPart)
-    assert part.reasoning == "reason"
-    assert part.details[0].type == "text"
-    assert part.details[0].text == "reason"
-    assert part.details[0].signature == "sig"
-
-
-def test_model_message_to_ui_message_binary_content():
-    """Test converting a ModelRequest with BinaryContent to UIMessage."""
-    bin_data = b"bin"
-    model_message = ModelRequest(
-        parts=[
-            UserPromptPart(
-                content=[
-                    "What do you see?",
-                    BinaryContent(media_type="application/octet-stream", data=bin_data),
-                ]
-            ),
-        ],
-        kind="request",
-    )
-
-    result = model_message_to_ui_message(model_message)
-    assert result.role == "user"
-    assert result.parts == [TextUIPart(type="text", text="What do you see?")]
-    assert result.experimental_attachments == [
-        Attachment(
-            name=None,
-            contentType="application/octet-stream",
-            url="data:application/octet-stream;base64,Ymlu",
-        ),
-    ]
-
-
-def test_model_message_to_ui_message_image_url():
-    """Test converting a ModelRequest with ImageUrl to UIMessage."""
-    model_message = ModelRequest(
-        parts=[
-            UserPromptPart(
-                content=[
-                    "What do you see?",
-                    ImageUrl(identifier="doc1.png", url="/media/documents/doc1.png"),
-                ]
-            ),
-        ],
-        kind="request",
-    )
-
-    result = model_message_to_ui_message(model_message)
-    assert result.role == "user"
-    assert result.parts == [TextUIPart(type="text", text="What do you see?")]
-    assert result.experimental_attachments == [
-        Attachment(
-            name="doc1.png",
-            contentType="image/png",
-            url="/media/documents/doc1.png",
-        ),
-    ]
-
-
-def test_model_message_to_ui_message_document_url():
-    """Test converting a ModelRequest with DocumentUrl to UIMessage."""
-    model_message = ModelRequest(
-        parts=[
-            UserPromptPart(
-                content=[
-                    "Summarize this",
-                    DocumentUrl(
-                        identifier="doc1.pdf",
-                        url="/media/documents/doc1.pdf",
-                        media_type="application/pdf",
-                    ),
-                ]
-            ),
-        ],
-        kind="request",
-    )
-
-    result = model_message_to_ui_message(model_message)
-    assert result.role == "user"
-    assert result.parts == [TextUIPart(type="text", text="Summarize this")]
-    assert result.experimental_attachments == [
-        Attachment(
-            name="doc1.pdf",
-            contentType="application/pdf",
-            url="/media/documents/doc1.pdf",
-        ),
-    ]
-
-
-def test_model_message_to_ui_message_file_parts_full():
-    """Test handling unsupported file parts in UserPromptPart content."""
-    for part_type in [AudioUrl, VideoUrl]:
-        model_message = ModelRequest(
-            parts=[
-                UserPromptPart(
-                    content=[
-                        "Check this file",
-                        part_type(url="http://example.com/file"),
-                    ],
-                    timestamp=None,
-                ),
-            ],
-            kind="request",
-        )
-
-        with pytest.raises(ValueError, match="Unsupported UserContent in UserPromptPart"):
-            model_message_to_ui_message(model_message)
-
-
-def test_model_message_to_ui_message_empty_parts():
-    """Test converting a ModelRequest with no valid parts returns None."""
-    model_message = ModelRequest(parts=[], kind="request")
-    assert model_message_to_ui_message(model_message) is None
-
-
-def test_model_message_to_ui_message_unsupported_part():
-    """Test handling unsupported part types in ModelRequest."""
-    model_message = ModelRequest(parts=[SystemPromptPart(content="sys")], kind="request")
-    assert model_message_to_ui_message(model_message) is None
-    model_message = ModelRequest(parts=[RetryPromptPart(content="retry")], kind="request")
-    assert model_message_to_ui_message(model_message) is None
-
-
-def test_model_message_to_ui_message_invalid_content_type():
-    """Test handling invalid content type in UserPromptPart."""
-
-    class DummyContent:
-        """Dummy class for testing invalid content types."""
-
-    model_message = ModelRequest(
-        parts=[UserPromptPart(content=[DummyContent()], timestamp=None)], kind="request"
-    )
-    with pytest.raises(ValueError, match="Unsupported UserContent in UserPromptPart"):
-        model_message_to_ui_message(model_message)
-
-
-def test_model_message_to_ui_message_invalid_response_part():
-    """Test handling invalid part type in ModelResponse."""
-
-    class DummyPart:
-        """Dummy class for testing invalid part types."""
-
-    model_message = ModelResponse(parts=[DummyPart()])
-    with pytest.raises(ValueError, match="Unsupported ModelMessage part type"):
-        model_message_to_ui_message(model_message)
-
-
-def test_model_message_to_ui_message_multiple_text_parts():
-    """Test converting a ModelResponse with multiple TextParts to UIMessage."""
-    model_message = ModelResponse(parts=[TextPart(content="A"), TextPart(content="B")])
-    result = model_message_to_ui_message(model_message)
-    assert result.role == "assistant"
-    parts_list = list(result.parts)
-    assert [p.text for p in parts_list if isinstance(p, TextUIPart)] == ["A", "B"]
-    assert result.content == "AB"
-
-
-def test_model_message_to_ui_message_userpromptpart_list_of_str():
-    """Test converting a ModelRequest with UserPromptPart containing list of strings."""
-    model_message = ModelRequest(
-        parts=[UserPromptPart(content=["A", "B"], timestamp=None)], kind="request"
-    )
-    result = model_message_to_ui_message(model_message)
-    assert result.role == "user"
-    parts_list = list(result.parts)
-    assert [p.text for p in parts_list if isinstance(p, TextUIPart)] == ["A", "B"]
-    assert result.content == "AB"
-
-
-def test_model_message_to_ui_message_tool_call_args_str():
-    """Test converting a ModelResponse with ToolCallPart containing JSON string args."""
-    args = {"foo": "bar"}
-    model_message = ModelResponse(
-        parts=[ToolCallPart(tool_call_id="id1", tool_name="tool", args=json.dumps(args))]
-    )
-    result = model_message_to_ui_message(model_message)
-    parts_list = list(result.parts)
-    part = parts_list[0]
-    assert isinstance(part, ToolInvocationUIPart)
-    assert part.toolInvocation.args == args
-
-
-def test_model_message_to_ui_message_with_reasoning_signature_none():
-    """Test converting a ModelResponse with ThinkingPart having signature=None."""
-    model_message = ModelResponse(parts=[ThinkingPart(content="reason", signature=None)])
-    result = model_message_to_ui_message(model_message)
-    parts_list = list(result.parts)
-    part = parts_list[0]
-    assert isinstance(part, ReasoningUIPart)
-    assert part.details[0].signature is None
-
-
-def test_model_message_to_ui_message_created_at_response():
-    """Test converting a ModelResponse with a specific timestamp."""
-    model_message = ModelResponse(
-        parts=[TextPart(content="Hi!")], timestamp=datetime.datetime(2024, 1, 1, 12, 0, 0)
-    )
-    result = model_message_to_ui_message(model_message)
-    assert result.createdAt == datetime.datetime(2024, 1, 1, 12, 0, 0)