✅(backend) test

i am testing the rescore feature

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,5 +7,5 @@ Description...
 
 Description...
 
-- [] item 1...
-- [] item 2...
+- [ ] item 1...
+- [ ] item 2...

.github/workflows/deploy.yml

@@ -1,52 +0,0 @@
-name: Deploy
-
-on:
-  push:
-    tags:
-      - 'preprod'
-      - 'production'
-
-
-jobs:
-  notify-argocd:
-    runs-on: ubuntu-latest
-    steps:
-      -
-        uses: actions/create-github-app-token@v1
-        id: app-token
-        with:
-          app-id: ${{ secrets.APP_ID }}
-          private-key: ${{ secrets.PRIVATE_KEY }}
-          owner: ${{ github.repository_owner }}
-          repositories: "drive,secrets"
-      -
-        name: Checkout repository
-        uses: actions/checkout@v2
-        with:
-          submodules: recursive
-          token: ${{ steps.app-token.outputs.token }}
-      -
-        name: Load sops secrets
-        uses: rouja/actions-sops@main
-        with:
-          secret-file: secrets/numerique-gouv/drive/secrets.enc.env
-          age-key: ${{ secrets.SOPS_PRIVATE }}
-      -
-        name: Call argocd github webhook
-        run: |
-          data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}'
-          sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}')
-          curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL
-          sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_PRODUCTION_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}')
-          curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_PRODUCTION_WEBHOOK_URL
-
-  start-test-on-preprod:
-    needs:
-      - notify-argocd
-    runs-on: ubuntu-latest
-    if: startsWith(github.event.ref, 'refs/tags/preprod')
-    steps:
-      -
-        name: Debug
-        run: |
-          echo "Start test when preprod is ready"

.github/workflows/docker-hub.yml

@@ -20,7 +20,13 @@ jobs:
     steps:
       -
         name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
       -
         name: Docker meta
         id: meta
@@ -44,6 +50,7 @@ jobs:
         with:
           context: .
           target: backend-production
+          platforms: linux/amd64,linux/arm64
           build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}

.github/workflows/find.yml

@@ -14,7 +14,7 @@ jobs:
     if: github.event_name == 'pull_request' # Makes sense only for pull requests
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
       - name: show
@@ -37,7 +37,7 @@ jobs:
       github.event_name == 'pull_request'
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         with:
           fetch-depth: 50
       - name: Check that the CHANGELOG has been modified in the current branch
@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v6
       - name: Check CHANGELOG max line length
         run: |
           max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L)
@@ -63,19 +63,22 @@ jobs:
         working-directory: src/backend
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
-      - name: Install Python
-        uses: actions/setup-python@v3
+        uses: actions/checkout@v6
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.12"
-      - name: Install development dependencies
-        run: pip install --user .[dev]
+          python-version-file: "src/backend/pyproject.toml"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+      - name: Install the project
+        run: uv sync --locked --all-extras
+
       - name: Check code formatting with ruff
-        run: ~/.local/bin/ruff format . --diff
+        run: uv run ruff format . --diff
       - name: Lint code with ruff
-        run: ~/.local/bin/ruff check .
+        run: uv run ruff check .
       - name: Lint code with pylint
-        run: ~/.local/bin/pylint .
+        run: uv run pylint .
 
   test-back:
     runs-on: ubuntu-latest
@@ -105,6 +108,11 @@ jobs:
           OPENSEARCH_INITIAL_ADMIN_PASSWORD: find.PASS123
         ports:
           - 9200:9200
+        options: >-
+          --health-cmd "curl -s http://localhost:9200 > /dev/null || exit 1"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 10
 
     env:
       DJANGO_CONFIGURATION: Test
@@ -123,20 +131,35 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Create writable /data
         run: |
           sudo mkdir -p /data/media && \
           sudo mkdir -p /data/static
 
-      - name: Install Python
-        uses: actions/setup-python@v3
+      - name: Set up Python
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.12"
+          python-version-file: "src/backend/pyproject.toml"
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
 
-      - name: Install development dependencies
-        run: pip install --user .[dev]
+      - name: Wait for OpenSearch to be ready
+        run: |
+          for i in {1..60}; do
+            if curl -s http://localhost:9200 > /dev/null; then
+              echo "OpenSearch is ready"
+              exit 0
+            fi
+            echo "Waiting for OpenSearch... attempt $i"
+            sleep 1
+          done
+          echo "OpenSearch failed to start"
+          exit 1
+
+      - name: Install the dependencies
+        run: uv sync --locked --all-extras
 
       - name: Run tests
-        run: ~/.local/bin/pytest
+        run: uv run pytest

.github/workflows/release-helm-chart.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         with:
           fetch-depth: 0
 

CHANGELOG.md

@@ -9,19 +9,39 @@ and this project adheres to
 # Unreleased
 
 ## Added
-
+- ✨(backend) add rescore on `updated_at`
+- 👷(docker) add arm64 platform support for image builds
 - ✨(backend) add semantic search
+- ✨(backend) add multi-embedding and chunking
+- ✨(backend) add analyzers to full-text search
+- ✨(backend) handle french, english, german and dutch 
+- ✨(backend) add evaluation command
 - backend application
 - helm chart
-- 🐛(backend) fix missing index creation in 'index/' view
 - ✨(backend) allow indexation of documents with either empty content or title.
 - ✨(api) new fulltext 'search/' view with OIDC resource server authentication
 - ✨(backend) limit access to documents : public & authenticated with a
-              linkreach & owned ones
-- ✨(backend) limit search to the calling app (audience) and a configured
               list of services
 - 🔧(compose) rename docker network 'lasuite-net' as 'lasuite-network'
 - ✨(backend) add demo service for Drive.
-- 🐛(backend) Fix parallel test execution issues
-- ✨(backend) Add OPENSEARCH_INDEX_PREFIX setting to prevent naming overlaping
+- ✨(backend) add OPENSEARCH_INDEX_PREFIX setting to prevent naming overlaping
               issues if the opensearch database is shared between apps.
+- ✨(backend) add tags
+- ✨(backend) adapt to conversation RAG
+- ✨(backend) add deletion endpoint
+- ✨(backend) add path filter
+- ✨(backend) add search_type param
+
+## Changed
+
+- 🏗️(backend) switch Python dependency management to uv
+- ✨(backend) allow deletion by tags
+- ♻️(backend) improve the evaluation command
+
+## Fixed
+
+- 🐛(backend) fix missing index creation in 'index/' view
+- 🐛(backend) fix parallel test execution issues
+
+## Removed
+- 🗑️(backend) remove sorting

Dockerfile

@@ -3,9 +3,6 @@
 # ---- base image to inherit from ----
 FROM python:3.12-slim-bookworm AS base
 
-# Upgrade pip to its latest release to speed up dependencies installation
-RUN python -m pip install --upgrade pip
-
 # Upgrade system packages to install security updates
 RUN apt-get update && \
   apt-get -y upgrade && \
@@ -14,13 +11,28 @@ RUN apt-get update && \
 # ---- Back-end builder image ----
 FROM base AS back-builder
 
-WORKDIR /builder
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LINK_MODE=copy
 
-# Copy required python dependencies
-COPY ./src/backend /builder
+# Disable Python downloads, because we want to use the system interpreter
+# across both images. If using a managed Python version, it needs to be
+# copied from the build image into the final image;
+ENV UV_PYTHON_DOWNLOADS=0
 
-RUN mkdir /install && \
-  pip install --prefix=/install .
+# install uv
+COPY --from=ghcr.io/astral-sh/uv:0.9.10 /uv /uvx /bin/
+
+WORKDIR /app
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=src/backend/uv.lock,target=uv.lock \
+    --mount=type=bind,source=src/backend/pyproject.toml,target=pyproject.toml \
+    uv sync --locked --no-install-project --no-dev
+
+COPY ./src/backend /app
+
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --locked --no-dev
 
 # ---- static link collector ----
 FROM base AS link-collector
@@ -33,11 +45,10 @@ RUN apt-get update && \
       rdfind && \
     rm -rf /var/lib/apt/lists/*
 
-# Copy installed python dependencies
-COPY --from=back-builder /install /usr/local
+# Copy the application from the builder
+COPY --from=back-builder /app /app
 
-# Copy find application (see .dockerignore)
-COPY ./src/backend /app/
+ENV PATH="/app/.venv/bin:$PATH"
 
 WORKDIR /app
 
@@ -76,14 +87,13 @@ COPY ./docker/files/usr/local/bin/entrypoint /usr/local/bin/entrypoint
 # docker user (see entrypoint).
 RUN chmod g=u /etc/passwd
 
-# Copy installed python dependencies
-COPY --from=back-builder /install /usr/local
-
-# Copy find application (see .dockerignore)
-COPY ./src/backend /app/
+# Copy the prepared application (see .dockerignore)
+COPY --from=back-builder /app /app
 
 WORKDIR /app
 
+ENV PATH="/app/.venv/bin:$PATH"
+
 # We wrap commands run in this container by the following entrypoint that
 # creates a user on-the-fly with the container user ID (see USER) and root group
 # ID.
@@ -100,10 +110,9 @@ RUN apt-get update && \
     apt-get install -y postgresql-client && \
     rm -rf /var/lib/apt/lists/*
 
-# Uninstall find and re-install it in editable mode along with development
-# dependencies
-RUN pip uninstall -y find
-RUN pip install -e .[dev]
+# Install development dependencies
+RUN --mount=from=ghcr.io/astral-sh/uv:0.9.10,source=/uv,target=/bin/uv \
+    uv sync --locked --all-extras
 
 # Restore the un-privileged user running the application
 ARG DOCKER_USER

Makefile

@@ -173,7 +173,7 @@ superuser: ## Create an admin superuser with password "admin"
 .PHONY: superuser
 
 back-i18n-compile: ## compile the gettext files
-	@$(MANAGE) compilemessages --ignore="venv/**/*"
+	@$(MANAGE) compilemessages --ignore=".venv/**/*"
 .PHONY: back-i18n-compile
 
 back-i18n-generate: ## create the .pot files used for i18n

UPGRADE.md

@@ -1,17 +0,0 @@
-# Upgrade
-
-All instructions to upgrade this project from one release to the next will be
-documented in this file. Upgrades must be run sequentially, meaning you should
-not skip minor/major releases while upgrading (fix releases can be skipped).
-
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
-and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
-
-For most upgrades, you just need to run the django migrations with
-the following command inside your docker container:
-
-`python manage.py migrate`
-
-(Note : in your development environment, you can `make migrate`.)
-
-## [Unreleased]

compose.yml

@@ -70,6 +70,7 @@ services:
     volumes:
       - ./src/backend:/app
       - ./data/static:/data/static
+      - /app/.venv
     depends_on:
       postgresql:
         condition: service_started
@@ -90,6 +91,7 @@ services:
     volumes:
       - ./src/backend:/app
       - ./data/static:/data/static
+      - /app/.venv
     depends_on:
       - app
 

docs/env.md

@@ -13,6 +13,8 @@ These are the environment variables you can set for the `find-backend` container
 | API_USERS_LIST_THROTTLE_RATE_SUSTAINED          | Throttle rate for api                                                                                                       | 180/hour                                                                |
 | CACHES_DEFAULT_TIMEOUT                          | Cache default timeout                                                                                                       | 30                                                                      |
 | CACHES_KEY_PREFIX                               | The prefix used to every cache keys.                                                                                        | docs                                                                    |
+| CHUNK_SIZE                                      | approximate number of characters of document content chunks                                                                 | 512                                                                     |
+| CHUNK_OVERLAP                                   | approximate number of characters of document content overlapping                                                            | 50                                                                      |
 | DB_ENGINE                                       | Engine to use for database connections                                                                                      | django.db.backends.postgresql_psycopg2                                  |
 | DB_HOST                                         | Host of the database                                                                                                        | localhost                                                               |
 | DB_NAME                                         | Name of the database                                                                                                        | impress                                                                 |
@@ -39,10 +41,17 @@ These are the environment variables you can set for the `find-backend` container
 | DJANGO_SECRET_KEY                               | Secret key                                                                                                                  |                                                                         |
 | DJANGO_SERVER_TO_SERVER_API_TOKENS              |                                                                                                                             | []                                                                      |
 | DOCUMENT_IMAGE_MAX_SIZE                         | Maximum size of document in bytes                                                                                           | 10485760                                                                |
+| EMBEDDING_API_KEY                               | API key of the embedding api                                                                                                |                                                                         |
+| EMBEDDING_API_MODEL_NAME                        | Name of the embedding model used on the api                                                                                 | embeddings-small                                                        |
+| EMBEDDING_API_PATH                              | URL of the embedding api                                                                                                    |                                                                         |
+| EMBEDDING_DIMENSION                             | Size of the embedding vector                                                                                                | 1024                                                                    |
+| EMBEDDING_REQUEST_TIMEOUT                       | time out in seconds of the embedding requests                                                                               | 10                                                                      |
 | FRONTEND_CSS_URL                                | To add a external css file to the app                                                                                       |                                                                         |
 | FRONTEND_HOMEPAGE_FEATURE_ENABLED               | Frontend feature flag to display the homepage                                                                               | false                                                                   |
 | FRONTEND_THEME                                  | Frontend theme to use                                                                                                       |                                                                         |
-| LANGUAGE_CODE                                   | Default language                                                                                                            | en-us                                                                   |
+| HYBRID_SEARCH_ENABLED                           | Flag to enable hybrid (an then semantic) search                                                                             | True                                                                    |
+| HYBRID_SEARCH_WEIGHTS                           | Weights used in the weighted sum of the hybrid search score                                                                 | [0.3, 0.7]                                                              |
+| LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD         | Language detection confidence threshold                                                                                     | 0.75                                                                    |
 | LOGGING_LEVEL_LOGGERS_APP                       | Application logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"                                         | INFO                                                                    |
 | LOGGING_LEVEL_LOGGERS_ROOT                      | Default logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"                                             | INFO                                                                    |
 | LOGIN_REDIRECT_URL                              | Login redirect url                                                                                                          |                                                                         |
@@ -100,13 +109,8 @@ These are the environment variables you can set for the `find-backend` container
 | THEME_CUSTOMIZATION_CACHE_TIMEOUT               | Cache duration for the customization settings                                                                               | 86400                                                                   |
 | THEME_CUSTOMIZATION_FILE_PATH                   | Full path to the file customizing the theme. An example is provided in src/backend/impress/configuration/theme/default.json | BASE_DIR/impress/configuration/theme/default.json                       |
 | TRASHBIN_CUTOFF_DAYS                            | Trashbin cutoff                                                                                                             | 30                                                                      |
+| TRIGRAMS_BOOST                                  | weight boost applied to trigram score in document matching score                                                            | 0.25                                                                    |
+| TRIGRAMS_MINIMUM_SHOULD_MATCH                   | minimal number or proportion of trigrams having to match to score                                                           | 0.75%                                                                   |
 | USER_OIDC_ESSENTIAL_CLAIMS                      | Essential claims in OIDC token                                                                                              | []                                                                      |
 | Y_PROVIDER_API_BASE_URL                         | Y Provider url                                                                                                              |                                                                         |
 | Y_PROVIDER_API_KEY                              | Y provider API key                                                                                                          |                                                                         |
-| HYBRID_SEARCH_ENABLED                           | Flag to enable hybrid (an then semantic) search                                                                             | True                                                                    |
-| HYBRID_SEARCH_WEIGHTS                           | Weights used in the weighted sum of the hybrid search score                                                                 | [0.3, 0.7]                                                              |
-| EMBEDDING_API_PATH                              | URL of the embedding api                                                                                                    | https://albert.api.etalab.gouv.fr/v1/embeddings                         |
-| EMBEDDING_API_KEY                               | API key of the embedding api                                                                                                |                                                                         |
-| EMBEDDING_REQUEST_TIMEOUT                       | time out in seconds of the embedding requests                                                                               | 10                                                                      |
-| EMBEDDING_API_MODEL_NAME                        | Name of the embedding model used on the api                                                                                 | embeddings-small                                                        |
-| EMBEDDING_DIMENSION                             | Size of the embedding vector                                                                                                | 1024                                                                    |

docs/evaluate-search-engine.md

@@ -0,0 +1,32 @@
+# Search Engine Evaluation Command
+
+## Overview 
+
+this Django command atomizes the evaluation of the search engine by computing 4 metrics: Average Discounted Cumulative Gain, Precision, Recall and F1 score. 
+
+## Usage 
+
+```
+python manage.py evaluate_search_engine <dataset_name> [options]
+```
+
+## Required Arguments
+- `dataset_name`: Name of the evaluation dataset to use. Datasets are located in `evaluation/management/commands/data/evaluation/`
+
+## Optional Arguments
+- `--min_score`: Minimum score threshold; hits below this score are ignored
+- `--keep-index`: Preserve the evaluation index after completion
+- `--force-reindex`: Drop and recreate the index even if it exists
+
+## Examples
+
+````
+# Basic evaluation with default settings
+python manage.py evaluate_search_engine my_dataset
+
+# Evaluation with minimum score threshold
+python manage.py evaluate_search_engine my_dataset --min_score 0.5
+
+# Force reindexing and clean up afterward
+python manage.py evaluate_search_engine my_dataset --force-reindex True --keep-index False
+````

docs/evaluation.md

@@ -0,0 +1,170 @@
+This file keeps for reference the logs of the last best evaluation of the model.
+
+````
+(venv) ➜  find git:(evaluate) ✗ docker compose exec app python manage.py evaluate_search_engine v1 --min_score 0.5   
+
+2025-11-20 18:44:30,903 core.services.opensearch INFO Hybrid search is disabled via HYBRID_SEARCH_ENABLED setting
+2025-11-20 18:44:31,183 core.management.commands.utils INFO Deleting search pipeline hybrid-search-pipeline
+2025-11-20 18:44:31,201 opensearch INFO DELETE http://opensearch:9200/_search/pipeline/hybrid-search-pipeline [status:200 request:0.018s]
+2025-11-20 18:44:31,202 opensearch WARNING GET http://opensearch:9200/_search/pipeline/hybrid-search-pipeline [status:404 request:0.001s]
+2025-11-20 18:44:31,202 core.management.commands.create_search_pipeline INFO Creating search pipeline: hybrid-search-pipeline
+2025-11-20 18:44:31,221 opensearch INFO PUT http://opensearch:9200/_search/pipeline/hybrid-search-pipeline [status:200 request:0.019s]
+2025-11-20 18:44:31,222 opensearch INFO HEAD http://opensearch:9200/evaluation-index [status:200 request:0.001s]
+[INFO] Starting evaluation with 76 documents and 12 queries
+2025-11-20 18:44:31,222 core.services.opensearch INFO embed: 'cours d'histoire de l'antiquité'
+2025-11-20 18:44:31,302 core.services.opensearch INFO Performing hybrid search with embedding: cours d'histoire de l'antiquité
+2025-11-20 18:44:31,317 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.014s]
+
+[QUERY EVALUATION]
+  q: cours d'histoire de l'antiquité
+  expect: ["L'Empire Romain", "L'Égypte Ancienne"]
+  result: ["L'Égypte Ancienne", 'La Sculpture sur Pierre']
+  NDCG: 61.31% 
+  PRECISION: 50.00% 
+  RECALL: 50.00% 
+  F1-SCORE: 50.00% 
+2025-11-20 18:44:31,317 core.services.opensearch INFO embed: 'recette salée végétarienne'
+2025-11-20 18:44:31,392 core.services.opensearch INFO Performing hybrid search with embedding: recette salée végétarienne
+2025-11-20 18:44:31,403 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.010s]
+
+[QUERY EVALUATION]
+  q: recette salée végétarienne
+  expect: ['Ratatouille Provençale', 'Salade de légumes', 'Fondue Savoyarde']
+  result: ['Salade de légumes']
+  NDCG: 46.93% 
+  PRECISION: 100.00% 
+  RECALL: 33.33% 
+  F1-SCORE: 50.00% 
+2025-11-20 18:44:31,403 core.services.opensearch INFO embed: 'art dramatique'
+2025-11-20 18:44:31,475 core.services.opensearch INFO Performing hybrid search with embedding: art dramatique
+2025-11-20 18:44:31,486 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.011s]
+
+[QUERY EVALUATION]
+  q: art dramatique
+  expect: ['Le Théâtre']
+  result: ['Le Théâtre', 'Le Vitrail']
+  NDCG: 100.00% 
+  PRECISION: 50.00% 
+  RECALL: 100.00% 
+  F1-SCORE: 66.67% 
+2025-11-20 18:44:31,487 core.services.opensearch INFO embed: 'art de bouger son corps'
+2025-11-20 18:44:31,556 core.services.opensearch INFO Performing hybrid search with embedding: art de bouger son corps
+2025-11-20 18:44:31,573 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.017s]
+
+[QUERY EVALUATION]
+  q: art de bouger son corps
+  expect: ['La Danse', 'La Danse Contemporaine']
+  result: ['La Danse', 'La Danse Contemporaine']
+  NDCG: 100.00% 
+  PRECISION: 100.00% 
+  RECALL: 100.00% 
+  F1-SCORE: 100.00% 
+2025-11-20 18:44:31,573 core.services.opensearch INFO embed: 'mammifères aquatiques'
+2025-11-20 18:44:31,641 core.services.opensearch INFO Performing hybrid search with embedding: mammifères aquatiques
+2025-11-20 18:44:31,654 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.013s]
+
+[QUERY EVALUATION]
+  q: mammifères aquatiques
+  expect: ['Le Dauphin', 'La Baleine à Bosse']
+  result: ['Le Dauphin', 'La Baleine à Bosse', 'Le Manchot Empereur', 'Le Requin Blanc', 'Le Paresseux']
+  NDCG: 100.00% 
+  PRECISION: 40.00% 
+  RECALL: 100.00% 
+  F1-SCORE: 57.14% 
+2025-11-20 18:44:31,654 core.services.opensearch INFO embed: 'insectes pollinisateurs'
+2025-11-20 18:44:31,733 core.services.opensearch INFO Performing hybrid search with embedding: insectes pollinisateurs
+2025-11-20 18:44:31,746 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.012s]
+
+[QUERY EVALUATION]
+  q: insectes pollinisateurs
+  expect: ["L'Abeille"]
+  result: ["L'Abeille", 'Le Caméléon', 'Le Papillon Monarque']
+  NDCG: 100.00% 
+  PRECISION: 33.33% 
+  RECALL: 100.00% 
+  F1-SCORE: 50.00% 
+2025-11-20 18:44:31,746 core.services.opensearch INFO embed: 'prédateur félin'
+2025-11-20 18:44:31,820 core.services.opensearch INFO Performing hybrid search with embedding: prédateur félin
+2025-11-20 18:44:31,834 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.014s]
+
+[QUERY EVALUATION]
+  q: prédateur félin
+  expect: ["Le Lion d'Afrique", 'Le Guépard']
+  result: ["Le Lion d'Afrique", 'Le Guépard', 'Le Requin Blanc', "L'éléphant", 'Le Hibou Grand-Duc', "L'Ours polaire", 'Le Serpent Python']
+  NDCG: 100.00% 
+  PRECISION: 28.57% 
+  RECALL: 100.00% 
+  F1-SCORE: 44.44% 
+2025-11-20 18:44:31,835 core.services.opensearch INFO embed: 'elephant'
+2025-11-20 18:44:31,906 core.services.opensearch INFO Performing hybrid search with embedding: elephant
+2025-11-20 18:44:31,920 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.013s]
+
+[QUERY EVALUATION]
+  q: elephant
+  expect: ["L'Éléphant d'Asie", "L'éléphant"]
+  result: ["L'éléphant", "L'Éléphant d'Asie"]
+  NDCG: 100.00% 
+  PRECISION: 100.00% 
+  RECALL: 100.00% 
+  F1-SCORE: 100.00% 
+2025-11-20 18:44:31,920 core.services.opensearch INFO embed: 'courir'
+2025-11-20 18:44:31,994 core.services.opensearch INFO Performing hybrid search with embedding: courir
+2025-11-20 18:44:32,010 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.015s]
+
+[QUERY EVALUATION]
+  q: courir
+  expect: ['Il va courir']
+  result: ['Il va courir']
+  NDCG: 100.00% 
+  PRECISION: 100.00% 
+  RECALL: 100.00% 
+  F1-SCORE: 100.00% 
+2025-11-20 18:44:32,011 core.services.opensearch INFO embed: 'football'
+2025-11-20 18:44:32,082 core.services.opensearch INFO Performing hybrid search with embedding: football
+2025-11-20 18:44:32,089 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.007s]
+
+[QUERY EVALUATION]
+  q: football
+  expect: ['Foot']
+  result: ['Foot']
+  NDCG: 100.00% 
+  PRECISION: 100.00% 
+  RECALL: 100.00% 
+  F1-SCORE: 100.00% 
+2025-11-20 18:44:32,089 core.services.opensearch INFO embed: 'couri'
+2025-11-20 18:44:32,156 core.services.opensearch INFO Performing hybrid search with embedding: couri
+2025-11-20 18:44:32,163 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.007s]
+
+[QUERY EVALUATION]
+  q: couri
+  expect: ['Il va courir']
+  result: ['Coq au Vin', 'Il va courir', 'Clafoutis aux Cerises']
+  NDCG: 63.09% 
+  PRECISION: 33.33% 
+  RECALL: 100.00% 
+  F1-SCORE: 50.00% 
+2025-11-20 18:44:32,164 core.services.opensearch INFO embed: 'courrir'
+2025-11-20 18:44:32,231 core.services.opensearch INFO Performing hybrid search with embedding: courrir
+2025-11-20 18:44:32,240 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.009s]
+
+[QUERY EVALUATION]
+  q: courrir
+  expect: ['Il va courir']
+  result: ['Il va courir']
+  NDCG: 100.00% 
+  PRECISION: 100.00% 
+  RECALL: 100.00% 
+  F1-SCORE: 100.00% 
+
+============================================================
+[SUMMARY] Average Performance
+============================================================
+  Average NDCG: 89.28%
+  Average Precision: 69.60%
+  Average Recall: 90.28%
+  Average F1-Score: 72.35%
+2025-11-20 18:44:32,241 core.management.commands.utils INFO Deleting search pipeline hybrid-search-pipeline
+2025-11-20 18:44:32,258 opensearch INFO DELETE http://opensearch:9200/_search/pipeline/hybrid-search-pipeline [status:200 request:0.017s]
+
+[SUCCESS] Evaluation completed
+````

docs/setup-indexer.md

@@ -26,9 +26,26 @@ OPENSEARCH_USE_SSL=True
 OPENSEARCH_INDEX_PREFIX=find
 ```
 
+### Language
+
+Language specific operations are applied to document titles and contents to improve search results. 
+The language is automatically detected  by Find.
+If the language can not be detected no language specific operation are applied and the indexing process is not affected.
+
+Find supports french, english, german and dutch. 
+
+The search process is not language specific, a query can get documents of any language.
+
+Language detection estimates a confidence between 0 and 1. If the confidence is below a threshold the language is considered unrecognized. 
+This threshold can be controlled with LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD environment variable.
+
+```python
+LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD=0.75
+```
+
 ### Semantic search
 
-Find offers a semantic search feature. You can either use pure full-text search or a hybrid full-text + semantic search. To enable the hybrid search, add the fallowing settings. 
+Find offers a semantic search feature. You can either use pure full-text search or a hybrid full-text + semantic search. To enable the hybrid search, add the following settings. 
 
 ```python
 # Enable flag
@@ -38,6 +55,8 @@ HYBRID_SEARCH_ENABLED = True
 HYBRID_SEARCH_WEIGHTS = 0.7,0.3
 
 # Embedding
+CHUNK_SIZE=512
+CHUNK_OVERLAP=50
 EMBEDDING_API_PATH = https://embedding.api.example.com/full/path/
 EMBEDDING_API_KEY = your-embedding-api-key
 EMBEDDING_REQUEST_TIMEOUT = 10
@@ -49,6 +68,27 @@ The hybrid search computes a score for full-text and semantic search and combine
 
 You need to use an embedding api similar to https://albert.api.etalab.gouv.fr/documentation#tag/Embeddings/operation/embeddings_v1_embeddings_post. 
 
+### document chunking
+
+The indexing process embeds documents by converting their content into vector representations (embeddings). When a document exceeds the character dimension defined by CHUNK_SIZE, it's divided into smaller segments (chunks), with each chunk embedded independently. Each chunk must be smaller than the embedding model's context window . 
+
+The chunking algorithm works recursively. It attempts to create the largest possible segments first, then subdivides them further if they still exceed the size limit defined by CHUNK_SIZE. Segments can share overlapping content between them (set CHUNK_OVERLAP=0 to disable overlapping). 
+
+During the search, the matching of a document is the matching of its best chunk.
+
+## trigrams
+
+Find uses trigrams to improve the robustness of the full text search engine to spelling variations and errors. It can be configured by two environment variables. 
+
+````
+TRIGRAMS_BOOST=0.25
+TRIGRAMS_MINIMUM_SHOULD_MATCH=0.75%
+````
+
+`TRIGRAMS_BOOST` is weight boost applied to the trigram score in the document matching. 
+`TRIGRAMS_MINIMUM_SHOULD_MATCH` is the minimal number or proportion of trigrams having to match to score. It is
+either an absolute number or proportion.
+
 ## Setup indexation API
 
 Other applications can index their files through the **`/index/`** endpoint with a simple token authentication.

env.d/development/common.dist

@@ -55,3 +55,7 @@ OIDC_RS_ENCRYPTION_KEY_TYPE="RSA"
 HYBRID_SEARCH_ENABLED=True
 EMBEDDING_API_KEY=ThisIsAnExampleKeyForDevPurposeOnly
 EMBEDDING_API_PATH=https://albert.api.etalab.gouv.fr/v1/embeddings
+
+## Multi-embedding: chunk documents and embed each chunk
+CHUNK_SIZE=512
+CHUNK_OVERLAP=50

src/backend/.pylintrc

@@ -7,7 +7,7 @@ extension-pkg-whitelist=
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=migrations
+ignore=migrations,.venv
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.
@@ -31,10 +31,6 @@ persistent=yes
 # Specify a configuration file.
 #rcfile=
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages
-suggestion-mode=yes
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no

src/backend/core/admin.py

@@ -1,8 +1,16 @@
 """Admin config for find's core app"""
 
-from django.contrib import admin
+from django.contrib import admin, messages
+from django.shortcuts import redirect, render
+from django.urls import path, reverse
 
+from core.management.commands.create_search_pipeline import (
+    ensure_search_pipeline_exists,
+)
+
+from . import selftests_builtin  # pylint: disable=unused-import
 from .models import Service
+from .selftests import registry
 
 
 @admin.register(Service)
@@ -14,3 +22,79 @@ class ServiceAdmin(admin.ModelAdmin):
     list_filter = ("is_active", "created_at")
     ordering = ("-created_at",)
     readonly_fields = ("created_at", "token")
+    change_list_template = "admin/core/services/change_list.html"
+
+    def get_urls(self):
+        urls = super().get_urls()
+        custom_urls = [
+            path(
+                "ensure-search-pipeline/",
+                self.admin_site.admin_view(self.ensure_search_pipeline_view),
+                name="core_service_ensure_search_pipeline",
+            ),
+        ]
+        return custom_urls + urls
+
+    def ensure_search_pipeline_view(self, request):
+        """Run the management command function to assert the pipeline exists."""
+        changelist_url = reverse("admin:core_service_changelist")
+
+        try:
+            ensure_search_pipeline_exists()
+        except Exception as exc:  # noqa: BLE001# pylint: disable=broad-exception-caught
+            self.message_user(
+                request, f"Failed to ensure search pipeline: {exc}", messages.ERROR
+            )
+        else:
+            self.message_user(
+                request, "Search pipeline presence insured", messages.SUCCESS
+            )
+
+        return redirect(changelist_url)
+
+
+def selftest_view(request):
+    """Display the self-test page and run tests if requested."""
+    # selftests_builtin and registry are imported at module level to ensure tests are registered
+    run_tests = request.GET.get("run", "false").lower() == "true"
+
+    if run_tests:
+        results = registry.run_all()
+        all_passed = all(result.success for result in results)
+    else:
+        results = []
+        all_passed = None
+
+    context = {
+        **admin.site.each_context(request),
+        "title": "System Self-Tests",
+        "results": results,
+        "all_passed": all_passed,
+        "run_tests": run_tests,
+        "available_tests": registry.get_all_tests(),
+    }
+
+    return render(request, "admin/selftest.html", context)
+
+
+# Add custom URL to the default admin site
+def get_admin_urls():
+    """Get URLs with selftest added."""
+
+    def get_urls():
+        urls = get_admin_urls.original_get_urls()
+        custom_urls = [
+            path(
+                "selftest/",
+                admin.site.admin_view(selftest_view),
+                name="selftest",
+            ),
+        ]
+        return custom_urls + urls
+
+    return get_urls
+
+
+# Store original get_urls and override it
+get_admin_urls.original_get_urls = admin.site.get_urls
+admin.site.get_urls = get_admin_urls()

src/backend/core/enums.py

@@ -13,6 +13,16 @@ class ReachEnum(str, Enum):
     RESTRICTED = "restricted"
 
 
+# Search type
+
+
+class SearchTypeEnum(str, Enum):
+    """Search type options"""
+
+    HYBRID = "hybrid"
+    FULL_TEXT = "full_text"
+
+
 # Fields
 
 CREATED_AT = "created_at"
@@ -21,12 +31,22 @@ PATH = "path"
 NUMCHILD = "numchild"
 REACH = "reach"
 SIZE = "size"
+TAGS = "tags"
 TITLE = "title"
+CONTENT = "content"
 UPDATED_AT = "updated_at"
 USERS = "users"
 GROUPS = "groups"
 
-RELEVANCE = "relevance"
-
-ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, REACH)
-SOURCE_FIELDS = (TITLE, SIZE, DEPTH, PATH, NUMCHILD, CREATED_AT, UPDATED_AT, REACH)
+SOURCE_FIELDS = (
+    TITLE,
+    CONTENT,
+    SIZE,
+    DEPTH,
+    PATH,
+    NUMCHILD,
+    CREATED_AT,
+    UPDATED_AT,
+    REACH,
+    TAGS,
+)

src/backend/core/factories.py

@@ -30,6 +30,7 @@ class DocumentSchemaFactory(factory.DictFactory):
     users = factory.LazyFunction(lambda: [str(uuid4()) for _ in range(3)])
     groups = factory.LazyFunction(lambda: [slugify(fake.word()) for _ in range(3)])
     reach = factory.Iterator(list(enums.ReachEnum))
+    tags = factory.LazyFunction(lambda: [])
     depth = 1
     numchild = 0
     is_active = True

src/backend/core/management/commands/create_search_pipeline.py

@@ -28,7 +28,7 @@ class Command(BaseCommand):
 def ensure_search_pipeline_exists():
     """Create search pipeline for hybrid search if it does not exist"""
     try:
-        opensearch_client().search_pipeline.get(settings.HYBRID_SEARCH_PIPELINE_ID)
+        opensearch_client().search_pipeline.get(id=settings.HYBRID_SEARCH_PIPELINE_ID)
         logger.info("Search pipeline exists already")
     except NotFoundError:
         logger.info("Creating search pipeline: %s", settings.HYBRID_SEARCH_PIPELINE_ID)
@@ -40,12 +40,13 @@ def ensure_search_pipeline_exists():
                 "phase_results_processors": [
                     {
                         "normalization-processor": {
+                            "normalization": {"technique": "min_max"},
                             "combination": {
                                 "technique": "arithmetic_mean",
                                 "parameters": {
                                     "weights": settings.HYBRID_SEARCH_WEIGHTS
                                 },
-                            }
+                            },
                         }
                     }
                 ],

src/backend/core/management/commands/reindex_with_embedding.py

@@ -10,12 +10,9 @@ from django.core.management.base import BaseCommand, CommandError
 from opensearchpy.exceptions import NotFoundError
 
 from core.models import get_opensearch_index_name
-from core.services.opensearch import (
-    check_hybrid_search_enabled,
-    embed_text,
-    format_document,
-    opensearch_client,
-)
+from core.services.indexing import chunk_document
+from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
+from core.utils import get_language_value
 
 logger = logging.getLogger(__name__)
 
@@ -60,7 +57,7 @@ def reindex_with_embedding(index_name, batch_size=500, scroll="10m"):
     returns a dict with the number of successful embeddings and failed embeddings.
     """
     opensearch_client_ = opensearch_client()
-    page = opensearch_client_.search(
+    page = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
         index=index_name,
         scroll=scroll,
         size=batch_size,
@@ -69,14 +66,27 @@ def reindex_with_embedding(index_name, batch_size=500, scroll="10m"):
             "query": {
                 "bool": {
                     "should": [
-                        {"bool": {"must_not": {"exists": {"field": "embedding"}}}},
                         {
                             "bool": {
-                                "must_not": {
-                                    "term": {
-                                        "embedding_model": settings.EMBEDDING_API_MODEL_NAME
+                                "must_not": [
+                                    {
+                                        "nested": {
+                                            "path": "chunks",
+                                            "query": {"match_all": {}},
+                                        }
                                     }
-                                }
+                                ]
+                            }
+                        },
+                        {
+                            "bool": {
+                                "must_not": [
+                                    {
+                                        "term": {
+                                            "embedding_model": settings.EMBEDDING_API_MODEL_NAME
+                                        }
+                                    }
+                                ]
                             }
                         },
                     ],
@@ -85,17 +95,17 @@ def reindex_with_embedding(index_name, batch_size=500, scroll="10m"):
             }
         },
     )
-
     nb_failed_embedding = 0
     nb_success_embedding = 0
     while len(page["hits"]["hits"]) > 0:
         actions = []
         for hit in page["hits"]["hits"]:
             source = hit["_source"]
-            embedding = embed_text(
-                format_document(source.get("title", ""), source.get("content", ""))
+            chunks = chunk_document(
+                get_language_value(source, "title"),
+                get_language_value(source, "content"),
             )
-            if embedding:
+            if chunks:
                 actions.append(
                     {
                         "update": {
@@ -110,7 +120,7 @@ def reindex_with_embedding(index_name, batch_size=500, scroll="10m"):
                 actions.append(
                     {
                         "doc": {
-                            "embedding": embedding,
+                            "chunks": chunks,
                             "embedding_model": settings.EMBEDDING_API_MODEL_NAME,
                         }
                     }
@@ -120,7 +130,9 @@ def reindex_with_embedding(index_name, batch_size=500, scroll="10m"):
                 nb_failed_embedding += 1
 
         opensearch_client_.bulk(index=index_name, body=actions)
-        page = opensearch_client_.scroll(scroll_id=page["_scroll_id"], scroll=scroll)
+        page = opensearch_client_.scroll(  # pylint: disable=unexpected-keyword-arg
+            scroll_id=page["_scroll_id"], scroll=scroll
+        )
 
     opensearch_client_.clear_scroll(scroll_id=page["_scroll_id"])
     return {

src/backend/core/schemas.py

@@ -1,6 +1,6 @@
 """Pydantic model to validate documents before indexation."""
 
-from typing import Annotated, List, Literal, Optional
+from typing import Annotated, List, Optional
 
 from django.utils import timezone
 from django.utils.text import slugify
@@ -18,6 +18,7 @@ from pydantic import (
 )
 
 from . import enums
+from .enums import SearchTypeEnum
 
 
 class DocumentSchema(BaseModel):
@@ -37,6 +38,7 @@ class DocumentSchema(BaseModel):
         default_factory=list
     )
     reach: Optional[enums.ReachEnum] = Field(default=enums.ReachEnum.RESTRICTED)
+    tags: List[Annotated[str, Field(max_length=100)]] = Field(default_factory=list)
     is_active: bool
 
     model_config = ConfigDict(
@@ -112,6 +114,25 @@ class SearchQueryParametersSchema(BaseModel):
     services: StringListParameter = Field(default_factory=list)
     visited: StringListParameter = Field(default_factory=list)
     reach: Optional[enums.ReachEnum] = None
-    order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default=enums.RELEVANCE)
-    order_direction: Optional[Literal["asc", "desc"]] = Field(default="desc")
+    tags: StringListParameter = Field(default_factory=list)
+    path: Optional[str] = None
     nb_results: Optional[conint(ge=1, le=300)] = Field(default=50)
+    search_type: Optional[SearchTypeEnum] = Field(default=None)
+    rescore: bool = Field(default=True)
+
+
+class DeleteDocumentsSchema(BaseModel):
+    """Schema for validating the delete documents request"""
+
+    service: str = Field(max_length=300)
+    document_ids: Optional[List[str]] = Field(default=None)
+    tags: Optional[List[str]] = Field(default=None)
+
+    @model_validator(mode="after")
+    def check_at_least_one_filter(self):
+        """Ensure at least one of document_ids or tags is provided"""
+        if not self.document_ids and not self.tags:
+            raise ValueError(
+                "At least one of 'document_ids' or 'tags' must be provided"
+            )
+        return self

src/backend/core/selftests.py

@@ -0,0 +1,125 @@
+"""
+Selftest registry and base classes for system health checks.
+
+This module provides a modular system for registering and running self-tests
+that check the health of various system components.
+"""
+
+import logging
+from typing import Dict, List, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class SelfTestResult:
+    """Result of a self-test execution."""
+
+    def __init__(  # pylint: disable=too-many-arguments,too-many-positional-arguments
+        self,
+        name: str,
+        success: bool,
+        message: str,
+        details: Optional[Dict] = None,
+        duration_ms: Optional[float] = None,
+    ):
+        self.name = name
+        self.success = success
+        self.message = message
+        self.details = details or {}
+        self.duration_ms = duration_ms
+
+    def to_dict(self) -> Dict:
+        """Convert result to dictionary."""
+        return {
+            "name": self.name,
+            "success": self.success,
+            "message": self.message,
+            "details": self.details,
+            "duration_ms": self.duration_ms,
+        }
+
+
+class SelfTest:
+    """Base class for self-tests."""
+
+    name: str = "Base Self Test"
+    description: str = "Base self-test class"
+
+    def run(self) -> SelfTestResult:
+        """
+        Execute the self-test.
+
+        Returns:
+            SelfTestResult: The result of the test execution.
+        """
+        raise NotImplementedError("Subclasses must implement the run method")
+
+
+class SelfTestRegistry:
+    """Registry for managing self-tests."""
+
+    def __init__(self):
+        self._tests: Dict[str, SelfTest] = {}
+
+    def register(self, test_class: type[SelfTest]) -> None:
+        """
+        Register a self-test class.
+
+        Args:
+            test_class: The SelfTest subclass to register.
+        """
+        test_instance = test_class()
+        test_id = test_class.__name__
+        if test_id in self._tests:
+            logger.warning("Self-test %s is already registered. Overwriting.", test_id)
+        self._tests[test_id] = test_instance
+        logger.debug("Registered self-test: %s - %s", test_id, test_instance.name)
+
+    def unregister(self, test_class: type[SelfTest]) -> None:
+        """
+        Unregister a self-test class.
+
+        Args:
+            test_class: The SelfTest subclass to unregister.
+        """
+        test_id = test_class.__name__
+        if test_id in self._tests:
+            del self._tests[test_id]
+            logger.debug("Unregistered self-test: %s", test_id)
+
+    def get_all_tests(self) -> List[SelfTest]:
+        """
+        Get all registered tests.
+
+        Returns:
+            List of registered SelfTest instances.
+        """
+        return list(self._tests.values())
+
+    def run_all(self) -> List[SelfTestResult]:
+        """
+        Run all registered tests.
+
+        Returns:
+            List of SelfTestResult objects.
+        """
+        results = []
+        for test in self._tests.values():
+            try:
+                result = test.run()
+                results.append(result)
+            except Exception as e:  # pylint: disable=broad-exception-caught
+                logger.exception("Error running self-test %s: %s", test.name, e)
+                results.append(
+                    SelfTestResult(
+                        name=test.name,
+                        success=False,
+                        message=f"Test failed with exception: {str(e)}",
+                        details={"exception": str(e)},
+                    )
+                )
+        return results
+
+
+# Global registry instance
+registry = SelfTestRegistry()

src/backend/core/selftests_builtin.py

@@ -0,0 +1,236 @@
+"""
+Built-in self-tests for core system components.
+
+This module contains self-tests for database, cache, and OpenSearch connectivity.
+"""
+
+import time
+
+from django.conf import settings
+from django.core.cache import cache
+from django.db import connection
+
+import sentry_sdk
+
+from .selftests import SelfTest, SelfTestResult, registry
+from .services.opensearch import opensearch_client
+
+
+class DatabaseSelfTest(SelfTest):
+    """Test database connectivity."""
+
+    name = "Database Connection"
+    description = "Verify that the database is accessible and responsive"
+
+    def run(self) -> SelfTestResult:
+        """Test database connection by executing a simple query."""
+        start_time = time.time()
+        try:
+            with connection.cursor() as cursor:
+                cursor.execute("SELECT 1")
+                result = cursor.fetchone()
+
+            duration_ms = (time.time() - start_time) * 1000
+
+            if result and result[0] == 1:
+                return SelfTestResult(
+                    name=self.name,
+                    success=True,
+                    message="Database connection successful",
+                    details={
+                        "database": settings.DATABASES["default"]["NAME"],
+                        "engine": settings.DATABASES["default"]["ENGINE"],
+                    },
+                    duration_ms=duration_ms,
+                )
+            return SelfTestResult(
+                name=self.name,
+                success=False,
+                message="Database query returned unexpected result",
+                duration_ms=duration_ms,
+            )
+        except (OSError, ValueError) as e:
+            duration_ms = (time.time() - start_time) * 1000
+            return SelfTestResult(
+                name=self.name,
+                success=False,
+                message=f"Database connection failed: {str(e)}",
+                details={"exception": str(e)},
+                duration_ms=duration_ms,
+            )
+
+
+class CacheSelfTest(SelfTest):
+    """Test cache (Redis) connectivity."""
+
+    name = "Cache Connection"
+    description = "Verify that the cache system is accessible and functional"
+
+    def run(self) -> SelfTestResult:
+        """Test cache by setting and getting a test value."""
+        start_time = time.time()
+        test_key = "selftest:cache:ping"
+        test_value = "pong"
+
+        try:
+            # Try to set a value
+            cache.set(test_key, test_value, timeout=10)
+
+            # Try to get the value back
+            retrieved_value = cache.get(test_key)
+
+            # Clean up
+            cache.delete(test_key)
+
+            duration_ms = (time.time() - start_time) * 1000
+
+            if retrieved_value == test_value:
+                cache_backend = settings.CACHES.get("default", {}).get(
+                    "BACKEND", "unknown"
+                )
+                return SelfTestResult(
+                    name=self.name,
+                    success=True,
+                    message="Cache connection successful",
+                    details={"backend": cache_backend},
+                    duration_ms=duration_ms,
+                )
+            return SelfTestResult(
+                name=self.name,
+                success=False,
+                message="Cache value mismatch",
+                details={
+                    "expected": test_value,
+                    "received": retrieved_value,
+                },
+                duration_ms=duration_ms,
+            )
+        except (OSError, ValueError, TimeoutError) as e:
+            duration_ms = (time.time() - start_time) * 1000
+            return SelfTestResult(
+                name=self.name,
+                success=False,
+                message=f"Cache connection failed: {str(e)}",
+                details={"exception": str(e)},
+                duration_ms=duration_ms,
+            )
+
+
+class OpenSearchSelfTest(SelfTest):
+    """Test OpenSearch connectivity."""
+
+    name = "OpenSearch Connection"
+    description = "Verify that OpenSearch is accessible and responsive"
+
+    def run(self) -> SelfTestResult:
+        """Test OpenSearch connection by checking cluster health."""
+        start_time = time.time()
+        try:
+            client = opensearch_client()
+
+            # Ping the cluster
+            if not client.ping():
+                duration_ms = (time.time() - start_time) * 1000
+                return SelfTestResult(
+                    name=self.name,
+                    success=False,
+                    message="OpenSearch ping failed",
+                    duration_ms=duration_ms,
+                )
+
+            # Get cluster health
+            health = client.cluster.health()
+
+            duration_ms = (time.time() - start_time) * 1000
+
+            return SelfTestResult(
+                name=self.name,
+                success=True,
+                message="OpenSearch connection successful",
+                details={
+                    "cluster_name": health.get("cluster_name", "unknown"),
+                    "status": health.get("status", "unknown"),
+                    "number_of_nodes": health.get("number_of_nodes", 0),
+                    "number_of_data_nodes": health.get("number_of_data_nodes", 0),
+                    "active_shards": health.get("active_shards", 0),
+                    "host": settings.OPENSEARCH_HOST,
+                    "port": settings.OPENSEARCH_PORT,
+                },
+                duration_ms=duration_ms,
+            )
+        except (OSError, ValueError, TimeoutError) as e:
+            duration_ms = (time.time() - start_time) * 1000
+            return SelfTestResult(
+                name=self.name,
+                success=False,
+                message=f"OpenSearch connection failed: {str(e)}",
+                details={"exception": str(e)},
+                duration_ms=duration_ms,
+            )
+
+
+class SentrySelfTest(SelfTest):
+    """Test Sentry connectivity."""
+
+    name = "Sentry Connection"
+    description = "Verify that Sentry is accessible and responsive"
+
+    def run(self) -> SelfTestResult:
+        """Test Sentry connection by sending a test error."""
+        if not sentry_sdk:
+            return SelfTestResult(
+                name=self.name,
+                success=False,
+                message="Sentry SDK not available",
+            )
+
+        # Check if Sentry DSN is configured
+        sentry_dsn = getattr(settings, "SENTRY_DSN", None)
+        if not sentry_dsn:
+            return SelfTestResult(
+                name=self.name,
+                success=False,
+                message="Sentry DSN not configured",
+            )
+
+        start_time = time.time()
+        try:
+            # Send a test error to Sentry to verify connectivity
+            scope = sentry_sdk.get_current_scope()
+            scope.set_extra("selftest", "Sentry connectivity test")
+
+            try:
+                # Raise a fake error that we'll catch and send to Sentry
+                raise ValueError(
+                    "Sentry self-test error - fake error for connectivity check"
+                )
+            except ValueError as test_error:
+                sentry_sdk.capture_exception(test_error)
+
+            duration_ms = (time.time() - start_time) * 1000
+
+            return SelfTestResult(
+                name=self.name,
+                success=True,
+                message="Sentry connection successful",
+                details={"dsn_configured": bool(sentry_dsn)},
+                duration_ms=duration_ms,
+            )
+        except Exception as e:  # noqa: BLE001 pylint: disable=broad-except
+            duration_ms = (time.time() - start_time) * 1000
+            return SelfTestResult(
+                name=self.name,
+                success=False,
+                message=f"Sentry connection failed: {str(e)}",
+                details={"exception": str(e)},
+                duration_ms=duration_ms,
+            )
+
+
+# Register all built-in tests
+registry.register(DatabaseSelfTest)
+registry.register(CacheSelfTest)
+registry.register(OpenSearchSelfTest)
+
+if settings.SENTRY_DSN:
+    registry.register(SentrySelfTest)

src/backend/core/services/embedding.py

@@ -0,0 +1,51 @@
+"""OpenSearch embedding utilities."""
+
+import logging
+
+from django.conf import settings
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+def embed_text(text):
+    """
+    Get embedding vector for the given text from any OpenAI-compatible embedding API
+    """
+    logger.info("embed: '%s'", text)
+
+    response = requests.post(
+        settings.EMBEDDING_API_PATH,
+        headers={"Authorization": f"Bearer {settings.EMBEDDING_API_KEY}"},
+        json={
+            "input": text,
+            "model": settings.EMBEDDING_API_MODEL_NAME,
+            "encoding_format": "float",
+        },
+        timeout=settings.EMBEDDING_REQUEST_TIMEOUT,
+    )
+
+    try:
+        response.raise_for_status()
+    except requests.HTTPError as e:
+        logger.warning("embedding API request failed: %s", str(e))
+        return None
+
+    try:
+        embedding = response.json()["data"][0]["embedding"]
+    except (KeyError, IndexError, TypeError):
+        logger.warning("unexpected embedding response format: %s", response.text)
+        return None
+
+    if len(embedding) != settings.EMBEDDING_DIMENSION:
+        logger.warning(
+            "unexpected embedding dimension: "
+            "EMBEDDING_DIMENSION is set to %d "
+            "but the configured embedding model returned a vector of dimension %d",
+            settings.EMBEDDING_DIMENSION,
+            len(embedding),
+        )
+        return None
+
+    return embedding

src/backend/core/services/indexing.py

@@ -0,0 +1,151 @@
+"""OpenSearch indexing utilities."""
+
+import logging
+
+from django.conf import settings
+from django.core.exceptions import SuspiciousOperation
+
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from opensearchpy.exceptions import NotFoundError
+from py3langid.langid import MODEL_FILE, LanguageIdentifier
+
+from core.services.opensearch_configuration import (
+    ANALYZERS,
+    FILTERS,
+    MAPPINGS,
+)
+
+from ..models import Service, get_opensearch_index_name
+from .embedding import embed_text
+from .opensearch import check_hybrid_search_enabled, opensearch_client
+
+logger = logging.getLogger(__name__)
+
+
+# see https://pypi.org/project/py3langid/
+LANGUAGE_IDENTIFIER = LanguageIdentifier.from_pickled_model(MODEL_FILE, norm_probs=True)
+LANGUAGE_IDENTIFIER.set_languages(["en", "fr", "de", "nl"])
+
+TEXT_SPLITER = RecursiveCharacterTextSplitter(
+    chunk_size=settings.CHUNK_SIZE,
+    chunk_overlap=settings.CHUNK_OVERLAP,
+)
+
+
+def ensure_index_exists(index_name):
+    """Create index if it does not exist"""
+    try:
+        opensearch_client().indices.get(index=index_name)
+    except NotFoundError:
+        logger.info("Creating index: %s", index_name)
+        opensearch_client().indices.create(
+            index=index_name,
+            body={
+                "settings": {
+                    "index.knn": True,
+                    "analysis": {
+                        "analyzer": ANALYZERS,
+                        "filter": FILTERS,
+                    },
+                },
+                "mappings": MAPPINGS,
+            },
+        )
+
+
+def prepare_document_for_indexing(document):
+    """Prepare document for indexing using nested language structure and handle embedding"""
+    language_code = detect_language_code(f"{document['title']} {document['content']}")
+    return {
+        "id": document["id"],
+        f"title.{language_code}": document["title"],
+        f"content.{language_code}": document["content"],
+        "embedding_model": settings.EMBEDDING_API_MODEL_NAME
+        if check_hybrid_search_enabled()
+        else None,
+        "chunks": chunk_document(
+            document["title"],
+            document["content"],
+        )
+        if check_hybrid_search_enabled()
+        else None,
+        "depth": document["depth"],
+        "path": document["path"],
+        "numchild": document["numchild"],
+        "created_at": document["created_at"],
+        "updated_at": document["updated_at"],
+        "size": document["size"],
+        "users": document["users"],
+        "groups": document["groups"],
+        "reach": document["reach"],
+        "tags": document.get("tags", []),
+        "is_active": document["is_active"],
+    }
+
+
+def chunk_document(title, content):
+    """
+    Chunk a document into multiple pieces and embed them.
+    """
+    chunks = []
+    for idx, chunked_content in enumerate(TEXT_SPLITER.split_text(content)):
+        embedding = embed_text(format_document(title, chunked_content))
+
+        if not embedding:
+            logger.warning(
+                "Failed to embed chunk %d of document '%s'. Document embedding is skipped",
+                idx,
+                title,
+            )
+            # if embedding fails for any chunk, we skip chunking the document
+            return None
+
+        chunks.append(
+            {
+                "index": idx,
+                "content": chunked_content,
+                "embedding": embedding,
+            }
+        )
+
+    logger.info("Document %s chunked into %d pieces", title, len(chunks))
+    return chunks
+
+
+def format_document(title, content):
+    """Get the embedding input format for a document"""
+    return f"<{title}>:<{content}>"
+
+
+def detect_language_code(text):
+    """Detect the language code of the document content."""
+
+    detected_code, confidence = LANGUAGE_IDENTIFIER.classify(text)
+
+    if confidence < settings.LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD:
+        return settings.UNDETERMINED_LANGUAGE_CODE
+
+    return detected_code
+
+
+def get_opensearch_indices(audience, services):
+    """
+    Get OpenSearch indices for the given audience and services.
+    """
+    try:
+        user_service = Service.objects.get(client_id=audience, is_active=True)
+    except Service.DoesNotExist as e:
+        logger.warning("Login failed: No service %s found", audience)
+        raise SuspiciousOperation("Service is not available") from e
+
+    # Find allowed sub-services for this service
+    allowed_services = set(user_service.services.values_list("name", flat=True))
+    allowed_services.add(user_service.name)
+
+    if services:
+        available_service = set(services).intersection(allowed_services)
+
+        if len(available_service) < len(services):
+            raise SuspiciousOperation("Some requested services are not available")
+
+    return [get_opensearch_index_name(service) for service in allowed_services]

src/backend/core/services/opensearch.py

@@ -1,17 +1,13 @@
-"""Opensearch related utils."""
+"""OpenSearch common utilities."""
 
 import logging
 from functools import cache
 
 from django.conf import settings
 
-import requests
 from opensearchpy import OpenSearch
-from opensearchpy.exceptions import NotFoundError
 from rest_framework.exceptions import ValidationError
 
-from core import enums
-
 logger = logging.getLogger(__name__)
 
 
@@ -46,269 +42,6 @@ def opensearch_client():
     )
 
 
-# pylint: disable=too-many-arguments, too-many-positional-arguments
-def search(  # noqa : PLR0913
-    q,
-    nb_results,
-    order_by,
-    order_direction,
-    search_indices,
-    reach,
-    visited,
-    user_sub,
-    groups,
-):
-    """Perform an OpenSearch search"""
-    query = get_query(
-        q=q,
-        nb_results=nb_results,
-        reach=reach,
-        visited=visited,
-        user_sub=user_sub,
-        groups=groups,
-    )
-    return opensearch_client().search(  # pylint: disable=unexpected-keyword-arg
-        index=",".join(search_indices),
-        body={
-            "_source": enums.SOURCE_FIELDS,  # limit the fields to return
-            "script_fields": {
-                "number_of_users": {"script": {"source": "doc['users'].size()"}},
-                "number_of_groups": {"script": {"source": "doc['groups'].size()"}},
-            },
-            "sort": get_sort(
-                query_keys=query.keys(),
-                order_by=order_by,
-                order_direction=order_direction,
-            ),
-            "size": nb_results,
-            # Compute query
-            "query": query,
-        },
-        params=get_params(query_keys=query.keys()),
-        # disable=unexpected-keyword-arg because
-        # ignore_unavailable is not in the the method declaration
-        ignore_unavailable=True,
-    )
-
-
-# pylint: disable=too-many-arguments, too-many-positional-arguments
-def get_query(  # noqa : PLR0913
-    q, nb_results, reach, visited, user_sub, groups
-):
-    """Build OpenSearch query body based on parameters"""
-    filter_ = get_filter(reach, visited, user_sub, groups)
-
-    if q == "*":
-        logger.info("Performing match_all query")
-        return {
-            "bool": {
-                "must": {"match_all": {}},
-                "filter": {"bool": {"filter": filter_}},
-            },
-        }
-
-    hybrid_search_enabled = check_hybrid_search_enabled()
-    if hybrid_search_enabled:
-        embedding = embed_text(q)
-    else:
-        embedding = None
-
-    if not embedding:
-        logger.info("Performing full-text search without embedding: %s", q)
-        return {
-            "bool": {
-                "must": {
-                    "multi_match": {
-                        "query": q,
-                        # Give title more importance over content by a power of 3
-                        "fields": ["title.text^3", "content"],
-                    }
-                },
-                "filter": filter_,
-            }
-        }
-
-    logger.info("Performing hybrid search with embedding: %s", q)
-    return {
-        "hybrid": {
-            "queries": [
-                {
-                    "bool": {
-                        "must": {
-                            "multi_match": {
-                                "query": q,
-                                # Give title more importance over content by a power of 3
-                                "fields": ["title.text^3", "content"],
-                            }
-                        },
-                        "filter": filter_,
-                    }
-                },
-                {
-                    "bool": {
-                        "must": {
-                            "knn": {
-                                "embedding": {
-                                    "vector": embedding,
-                                    "k": nb_results,
-                                }
-                            }
-                        },
-                        "filter": filter_,
-                    }
-                },
-            ]
-        }
-    }
-
-
-def get_filter(reach, visited, user_sub, groups):
-    """Build OpenSearch filter"""
-    filters = [
-        {"term": {"is_active": True}},  # filter out inactive documents
-        # Access control filters
-        {
-            "bool": {
-                "should": [
-                    # Public or authenticated (not restricted)
-                    {
-                        "bool": {
-                            "must_not": {
-                                "term": {enums.REACH: enums.ReachEnum.RESTRICTED},
-                            },
-                            "must": {
-                                "terms": {"_id": sorted(visited)},
-                            },
-                        }
-                    },
-                    # Restricted: either user or group must match
-                    {"term": {enums.USERS: user_sub}},
-                    {"terms": {enums.GROUPS: groups}},
-                ],
-                "minimum_should_match": 1,
-            }
-        },
-    ]
-
-    # Optional reach filter
-    if reach is not None:
-        filters.append({"term": {enums.REACH: reach}})
-
-    return filters
-
-
-def get_sort(query_keys, order_by, order_direction):
-    """Build OpenSearch sort clause"""
-    # Add sorting logic based on relevance or specified field
-    if "hybrid" in query_keys:
-        # sorting by other field than "_score" is not supported in hybrid search
-        # see: https://github.com/opensearch-project/neural-search/issues/866
-        return {"_score": {"order": order_direction}}
-    if order_by == enums.RELEVANCE:
-        return {"_score": {"order": order_direction}}
-
-    return {order_by: {"order": order_direction}}
-
-
-def get_params(query_keys):
-    """Build OpenSearch search parameters"""
-    if "hybrid" in query_keys:
-        return {"search_pipeline": settings.HYBRID_SEARCH_PIPELINE_ID}
-    return {}
-
-
-def embed_document(document):
-    """Get embedding vector for a given document"""
-    return embed_text(format_document(document.title, document.content))
-
-
-def format_document(title, content):
-    """Get the embedding input format for a document"""
-    return f"<{title}>:<{content}>"
-
-
-def embed_text(text):
-    """
-    Get embedding vector for the given text from any OpenAI-compatible embedding API
-    """
-    response = requests.post(
-        settings.EMBEDDING_API_PATH,
-        headers={"Authorization": f"Bearer {settings.EMBEDDING_API_KEY}>"},
-        json={
-            "input": text,
-            "model": settings.EMBEDDING_API_MODEL_NAME,
-            "dimensions": settings.EMBEDDING_DIMENSION,
-            "encoding_format": "float",
-        },
-        timeout=settings.EMBEDDING_REQUEST_TIMEOUT,
-    )
-
-    try:
-        response.raise_for_status()
-    except requests.HTTPError as e:
-        logger.warning("embedding API request failed: %s", str(e))
-        return None
-
-    try:
-        embedding = response.json()["data"][0]["embedding"]
-    except (KeyError, IndexError, TypeError):
-        logger.warning("unexpected embedding response format: %s", response.text)
-        return None
-
-    return embedding
-
-
-def ensure_index_exists(index_name):
-    """Create index if it does not exist"""
-    try:
-        opensearch_client().indices.get(index=index_name)
-    except NotFoundError:
-        logger.info("Creating index: %s", index_name)
-        opensearch_client().indices.create(
-            index=index_name,
-            body={
-                "settings": {"index.knn": True},
-                "mappings": {
-                    "dynamic": "strict",
-                    "properties": {
-                        "id": {"type": "keyword"},
-                        "title": {
-                            "type": "keyword",
-                            "fields": {"text": {"type": "text"}},
-                        },
-                        "depth": {"type": "integer"},
-                        "path": {
-                            "type": "keyword",
-                            "fields": {"text": {"type": "text"}},
-                        },
-                        "numchild": {"type": "integer"},
-                        "content": {"type": "text"},
-                        "created_at": {"type": "date"},
-                        "updated_at": {"type": "date"},
-                        "size": {"type": "long"},
-                        "users": {"type": "keyword"},
-                        "groups": {"type": "keyword"},
-                        "reach": {"type": "keyword"},
-                        "is_active": {"type": "boolean"},
-                        "embedding": {
-                            # for simplicity, embedding is always present but is empty
-                            # when hybrid search is disabled
-                            "type": "knn_vector",
-                            "dimension": settings.EMBEDDING_DIMENSION,
-                            "method": {
-                                "engine": "lucene",
-                                "space_type": "l2",
-                                "name": "hnsw",
-                                "parameters": {},
-                            },
-                        },
-                        "embedding_model": {"type": "keyword"},
-                    },
-                },
-            },
-        )
-
-
 @cache
 def check_hybrid_search_enabled():
     """Check that all required environment variables are set for hybrid search."""

src/backend/core/services/opensearch_configuration.py

@@ -0,0 +1,292 @@
+"""OpenSearch configuration."""
+
+from django.conf import settings
+
+ANALYZERS = {
+    "french_analyzer": {
+        "type": "custom",
+        "tokenizer": "standard",
+        "filter": [
+            "lowercase",
+            "asciifolding",
+            "french_elision",
+            "french_stop",
+            "french_stemmer",
+        ],
+    },
+    "english_analyzer": {
+        "type": "custom",
+        "tokenizer": "standard",
+        "filter": [
+            "lowercase",
+            "asciifolding",
+            "english_stop",
+            "english_stemmer",
+        ],
+    },
+    "german_analyzer": {
+        "type": "custom",
+        "tokenizer": "standard",
+        "filter": [
+            "lowercase",
+            "asciifolding",
+            "german_stop",
+            "german_stemmer",
+        ],
+    },
+    "dutch_analyzer": {
+        "type": "custom",
+        "tokenizer": "standard",
+        "filter": [
+            "lowercase",
+            "asciifolding",
+            "dutch_stop",
+            "dutch_stemmer",
+        ],
+    },
+    "undetermined_language_analyzer": {
+        "type": "custom",
+        "tokenizer": "standard",
+        "filter": [
+            "lowercase",
+            "asciifolding",
+        ],
+    },
+    "trigram_analyzer": {
+        "type": "custom",
+        "tokenizer": "standard",
+        "filter": [
+            "lowercase",
+            "asciifolding",
+            "trigram_filter",
+        ],
+    },
+}
+
+FILTERS = {
+    "french_elision": {
+        "type": "elision",
+        "articles_case": True,
+        "articles": [
+            "l",
+            "m",
+            "t",
+            "qu",
+            "n",
+            "s",
+            "j",
+            "d",
+            "c",
+            "jusqu",
+            "quoiqu",
+            "lorsqu",
+            "puisqu",
+        ],
+    },
+    "french_stop": {
+        "type": "stop",
+        "stopwords": "_french_",
+    },
+    "french_stemmer": {
+        "type": "stemmer",
+        "language": "light_french",
+    },
+    "english_stop": {
+        "type": "stop",
+        "stopwords": "_english_",
+    },
+    "english_stemmer": {
+        "type": "stemmer",
+        "language": "english",
+    },
+    "german_stop": {
+        "type": "stop",
+        "stopwords": "_german_",
+    },
+    "german_stemmer": {
+        "type": "stemmer",
+        "language": "light_german",
+    },
+    "dutch_stop": {
+        "type": "stop",
+        "stopwords": "_dutch_",
+    },
+    "dutch_stemmer": {
+        "type": "stemmer",
+        "language": "dutch",
+    },
+    "trigram_filter": {
+        "type": "ngram",
+        "min_gram": 3,
+        "max_gram": 3,
+    },
+}
+
+MAPPINGS = {
+    "dynamic": "strict",
+    "properties": {
+        "id": {"type": "keyword"},
+        # French
+        "title.fr": {
+            "type": "keyword",
+            "fields": {
+                "text": {
+                    "type": "text",
+                    "analyzer": "french_analyzer",
+                    "fields": {
+                        "trigrams": {
+                            "type": "text",
+                            "analyzer": "trigram_analyzer",
+                        }
+                    },
+                }
+            },
+        },
+        "content.fr": {
+            "type": "text",
+            "analyzer": "french_analyzer",
+            "fields": {
+                "trigrams": {
+                    "type": "text",
+                    "analyzer": "trigram_analyzer",
+                }
+            },
+        },
+        # English
+        "title.en": {
+            "type": "keyword",
+            "fields": {
+                "text": {
+                    "type": "text",
+                    "analyzer": "english_analyzer",
+                    "fields": {
+                        "trigrams": {
+                            "type": "text",
+                            "analyzer": "trigram_analyzer",
+                        }
+                    },
+                }
+            },
+        },
+        "content.en": {
+            "type": "text",
+            "analyzer": "english_analyzer",
+            "fields": {
+                "trigrams": {
+                    "type": "text",
+                    "analyzer": "trigram_analyzer",
+                }
+            },
+        },
+        # German
+        "title.de": {
+            "type": "keyword",
+            "fields": {
+                "text": {
+                    "type": "text",
+                    "analyzer": "german_analyzer",
+                    "fields": {
+                        "trigrams": {
+                            "type": "text",
+                            "analyzer": "trigram_analyzer",
+                        }
+                    },
+                }
+            },
+        },
+        "content.de": {
+            "type": "text",
+            "analyzer": "german_analyzer",
+            "fields": {
+                "trigrams": {
+                    "type": "text",
+                    "analyzer": "trigram_analyzer",
+                }
+            },
+        },
+        # Dutch
+        "title.nl": {
+            "type": "keyword",
+            "fields": {
+                "text": {
+                    "type": "text",
+                    "analyzer": "dutch_analyzer",
+                    "fields": {
+                        "trigrams": {
+                            "type": "text",
+                            "analyzer": "trigram_analyzer",
+                        }
+                    },
+                }
+            },
+        },
+        "content.nl": {
+            "type": "text",
+            "analyzer": "dutch_analyzer",
+            "fields": {
+                "trigrams": {
+                    "type": "text",
+                    "analyzer": "trigram_analyzer",
+                }
+            },
+        },
+        # Undetermined language
+        "title.und": {
+            "type": "keyword",
+            "fields": {
+                "text": {
+                    "type": "text",
+                    "analyzer": "undetermined_language_analyzer",
+                    "fields": {
+                        "trigrams": {
+                            "type": "text",
+                            "analyzer": "trigram_analyzer",
+                        }
+                    },
+                }
+            },
+        },
+        "content.und": {
+            "type": "text",
+            "analyzer": "undetermined_language_analyzer",
+            "fields": {
+                "trigrams": {
+                    "type": "text",
+                    "analyzer": "trigram_analyzer",
+                }
+            },
+        },
+        "depth": {"type": "integer"},
+        "path": {
+            "type": "keyword",
+            "fields": {"text": {"type": "text"}},
+        },
+        "numchild": {"type": "integer"},
+        "created_at": {"type": "date"},
+        "updated_at": {"type": "date"},
+        "size": {"type": "long"},
+        "users": {"type": "keyword"},
+        "groups": {"type": "keyword"},
+        "reach": {"type": "keyword"},
+        "tags": {"type": "keyword"},
+        "is_active": {"type": "boolean"},
+        "chunks": {
+            "type": "nested",
+            "properties": {
+                "index": {"type": "integer"},
+                "content": {"type": "text"},
+                "embedding": {
+                    "type": "knn_vector",
+                    "dimension": settings.EMBEDDING_DIMENSION,
+                    "method": {
+                        "engine": "lucene",
+                        "space_type": "l2",
+                        "name": "hnsw",
+                        "parameters": {},
+                    },
+                },
+            },
+        },
+        "embedding_model": {"type": "keyword"},
+    },
+}

src/backend/core/services/search.py

@@ -0,0 +1,260 @@
+"""OpenSearch search utilities."""
+
+import logging
+
+from django.conf import settings
+
+from core import enums
+from core.enums import SearchTypeEnum
+
+from .embedding import embed_text
+from .opensearch import check_hybrid_search_enabled, opensearch_client
+
+logger = logging.getLogger(__name__)
+
+
+# pylint: disable=too-many-arguments, too-many-positional-arguments
+def search(  # noqa : PLR0913
+    q,
+    nb_results,
+    search_indices,
+    reach,
+    visited,
+    user_sub,
+    groups,
+    tags,
+    search_type,
+    path=None,
+    rescore=False,
+):
+    """Perform an OpenSearch search"""
+    query = get_query(
+        q=q,
+        nb_results=nb_results,
+        reach=reach,
+        visited=visited,
+        user_sub=user_sub,
+        groups=groups,
+        tags=tags,
+        path=path,
+        search_type=search_type,
+    )
+    return opensearch_client().search(  # pylint: disable=unexpected-keyword-arg
+        index=",".join(search_indices),
+        body={
+            "_source": enums.SOURCE_FIELDS,  # limit the fields to return
+            "script_fields": {
+                "number_of_users": {"script": {"source": "doc['users'].size()"}},
+                "number_of_groups": {"script": {"source": "doc['groups'].size()"}},
+            },
+            "size": nb_results,
+            "query": query,
+            "rescore": get_rescore(nb_results=nb_results) if rescore else [],
+        },
+        params=get_params(query_keys=query.keys()),
+        # disable=unexpected-keyword-arg because
+        # ignore_unavailable is not in the method declaration
+        ignore_unavailable=True,
+    )
+
+
+# pylint: disable=too-many-arguments, too-many-positional-arguments
+def get_query(  # noqa : PLR0913
+    q,
+    nb_results,
+    reach,
+    visited,
+    user_sub,
+    groups,
+    tags,
+    search_type,
+    path=None,
+):
+    """Build OpenSearch query body based on parameters"""
+    filter_ = get_filter(reach, visited, user_sub, groups, tags, path)
+
+    if q == "*":
+        logger.info("Performing match_all query")
+        return {
+            "bool": {
+                "must": {"match_all": {}},
+                "filter": {"bool": {"filter": filter_}},
+            },
+        }
+
+    q_vector = vectorize_query(q, search_type)
+
+    if not q_vector:
+        logger.info("Performing full-text search without embedding: %s", q)
+        return get_full_text_query(q, filter_)
+
+    logger.info("Performing hybrid search with embedding: %s", q)
+    return {
+        "hybrid": {
+            "queries": [
+                get_full_text_query(q, filter_),
+                get_semantic_search_query(q_vector, filter_, nb_results),
+            ],
+        }
+    }
+
+
+def vectorize_query(q, search_type):
+    """Vectorize the query if hybrid search is enabled and requested"""
+    hybrid_search_enabled = check_hybrid_search_enabled()
+
+    if search_type == SearchTypeEnum.HYBRID:
+        if not hybrid_search_enabled:
+            logger.warning(
+                "Hybrid search was requested (search_type=hybrid) but is disabled on server",
+            )
+            return None
+
+        return embed_text(q)
+
+    return None
+
+
+def get_semantic_search_query(q_vector, filter_, nb_results):
+    """Build OpenSearch semantic search query"""
+    return {
+        "bool": {
+            "must": {
+                "nested": {
+                    "path": "chunks",
+                    "score_mode": "max",
+                    "query": {
+                        "knn": {
+                            "chunks.embedding": {
+                                "vector": q_vector,
+                                "k": nb_results,
+                            }
+                        }
+                    },
+                }
+            },
+            "filter": filter_,
+        }
+    }
+
+
+def get_full_text_query(q, filter_):
+    """Build OpenSearch full-text query"""
+    return {
+        "bool": {
+            "must": {
+                "bool": {
+                    "should": [
+                        {
+                            "multi_match": {
+                                "query": q,
+                                "fields": [
+                                    "title.*.text^3",
+                                    "content.*",
+                                ],
+                            }
+                        },
+                        {
+                            "multi_match": {
+                                "query": q,
+                                "fields": [
+                                    "title.*.text.trigrams^3",
+                                    "content.*.trigrams",
+                                ],
+                                "boost": settings.TRIGRAMS_BOOST,
+                                "minimum_should_match": settings.TRIGRAMS_MINIMUM_SHOULD_MATCH,
+                            }
+                        },
+                    ],
+                    "minimum_should_match": 1,
+                },
+            },
+            "filter": filter_,
+        }
+    }
+
+
+# pylint: disable=too-many-arguments, too-many-positional-arguments
+def get_filter(  # noqa : PLR0913
+    reach, visited, user_sub, groups, tags, path=None
+):
+    """Build OpenSearch filter"""
+    filters = [
+        {"term": {"is_active": True}},  # filter out inactive documents
+        # Access control filters
+        {
+            "bool": {
+                "should": [
+                    # Public or authenticated (not restricted)
+                    {
+                        "bool": {
+                            "must_not": {
+                                "term": {enums.REACH: enums.ReachEnum.RESTRICTED},
+                            },
+                            "must": {
+                                "terms": {"_id": sorted(visited)},
+                            },
+                        }
+                    },
+                    # Restricted: either user or group must match
+                    {"term": {enums.USERS: user_sub}},
+                    {"terms": {enums.GROUPS: groups}},
+                ],
+                "minimum_should_match": 1,
+            }
+        },
+    ]
+
+    # Optional reach filter
+    if reach is not None:
+        filters.append({"term": {enums.REACH: reach}})
+
+    # Optional tags filter
+    if tags:
+        # logical or: if tags are provided the matching documents should have at least one of them
+        filters.append({"terms": {"tags": tags}})
+
+    # Optional path filter
+    if path:
+        # filter documents that start with the provided path
+        filters.append({"prefix": {"path": path}})
+
+    return filters
+
+
+def get_rescore(nb_results):
+    """
+    Build rescore query.
+    Rescore is based on the `updated_at` field to boost more recently updated documents
+    """
+    return [
+        {
+            "window_size": nb_results,
+            "query": {
+                "rescore_query_weight": settings.RESCORE_UPDATED_AT_WEIGHT,
+                "rescore_query": {
+                    "function_score": {
+                        "functions": [
+                            {
+                                "gauss": {
+                                    "updated_at": {
+                                        "origin": "now",
+                                        "offset": settings.RESCORE_UPDATED_AT_OFFSET,
+                                        "scale": settings.RESCORE_UPDATED_AT_SCALE,
+                                        "decay": settings.RESCORE_UPDATED_AT_DECAY,
+                                    }
+                                }
+                            }
+                        ],
+                    }
+                },
+            },
+        }
+    ]
+
+
+def get_params(query_keys):
+    """Build OpenSearch search parameters"""
+    if "hybrid" in query_keys:
+        return {"search_pipeline": settings.HYBRID_SEARCH_PIPELINE_ID}
+    return {}

src/backend/core/templates/admin/core/services/change_list.html

@@ -0,0 +1,13 @@
+{% extends "admin/change_list.html" %}
+
+{% load i18n admin_urls %}
+
+{% block object-tools-items %}
+  <li>
+    <a href="{% url 'admin:core_service_ensure_search_pipeline' %}">
+      {% trans "Ensure search pipeline" %}
+    </a>
+  </li>
+  {{ block.super }}
+{% endblock %}
+

src/backend/core/templates/admin/index.html

@@ -0,0 +1,26 @@
+{% extends "admin/index.html" %}
+{% load i18n %}
+
+{% block sidebar %}
+{{ block.super }}
+
+<div id="content-main">
+    <div class="module" style="margin-top: 20px;">
+        <table>
+            <caption>
+                {% trans 'System Tools' %}
+            </caption>
+            <tbody>
+                <tr>
+                    <th scope="row">
+                        <a href="{% url 'admin:selftest' %}">{% trans 'System Self-Tests' %}</a>
+                    </th>
+                    <td>{% trans 'Run selftest checks' %}</td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+</div>
+
+{% endblock %}
+

src/backend/core/templates/admin/selftest.html

@@ -0,0 +1,258 @@
+{% extends "admin/base_site.html" %}
+{% load i18n static %}
+
+{% block extrastyle %}
+    {{ block.super }}
+    <style>
+        .selftest-container {
+            margin: 20px;
+        }
+
+        .selftest-module {
+            background: var(--body-bg);
+            border: 1px solid var(--border-color, #ddd);
+            border-radius: 4px;
+            margin-bottom: 20px;
+        }
+
+        .selftest-module h2 {
+            background: var(--primary);
+            color: var(--primary-fg);
+            padding: 10px 15px;
+            margin: 0;
+            font-size: 14px;
+            font-weight: normal;
+        }
+
+        .module-content {
+            padding: 15px;
+        }
+
+        .run-tests-btn {
+            display: inline-block;
+            padding: 10px 20px;
+            background: var(--primary);
+            color: var(--primary-fg);
+            text-decoration: none;
+            border-radius: 4px;
+            font-weight: bold;
+            margin-bottom: 20px;
+            border: none;
+            cursor: pointer;
+        }
+
+        .run-tests-btn:hover {
+            background: var(--primary-hover, #205067);
+        }
+
+        .overall-status {
+            padding: 15px;
+            border-radius: 4px;
+            font-weight: bold;
+            text-align: center;
+        }
+
+        .overall-status.success {
+            background-color: var(--success-bg, #d4edda);
+            color: var(--success-fg, #155724);
+            border: 2px solid var(--success-border, #c3e6cb);
+        }
+
+        .overall-status.failure {
+            background-color: var(--error-bg, #f8d7da);
+            color: var(--error-fg, #721c24);
+            border: 2px solid var(--error-border, #f5c6cb);
+        }
+
+        .test-item {
+            border: 1px solid var(--border-color, #ddd);
+            border-radius: 4px;
+            margin-bottom: 15px;
+            overflow: hidden;
+            background: var(--body-bg);
+        }
+
+        .test-header {
+            padding: 12px 15px;
+            background-color: var(--darkened-bg);
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            border-bottom: 1px solid var(--border-color, #ddd);
+        }
+
+        .test-name {
+            font-weight: bold;
+            font-size: 14px;
+            color: var(--body-fg);
+        }
+
+        .test-status {
+            padding: 4px 12px;
+            border-radius: 3px;
+            font-size: 11px;
+            font-weight: bold;
+            text-transform: uppercase;
+        }
+
+        .test-status.success {
+            background-color: #28a745;
+            color: white;
+        }
+
+        .test-status.failure {
+            background-color: #dc3545;
+            color: white;
+        }
+
+        .test-body {
+            padding: 15px;
+            background-color: var(--body-bg);
+        }
+
+        .test-message {
+            margin-bottom: 10px;
+            color: var(--body-fg);
+        }
+
+        .test-details {
+            background-color: var(--darkened-bg);
+            padding: 10px;
+            border-radius: 4px;
+            font-family: "Courier New", Monaco, monospace;
+            font-size: 12px;
+            border: 1px solid var(--border-color, #ddd);
+        }
+
+        .test-details dl {
+            margin: 0;
+        }
+
+        .test-details dt {
+            font-weight: bold;
+            color: var(--body-fg);
+            margin-top: 5px;
+        }
+
+        .test-details dd {
+            margin-left: 20px;
+            color: var(--body-quiet-color);
+            margin-bottom: 5px;
+        }
+
+        .duration {
+            color: var(--body-quiet-color);
+            font-size: 11px;
+            font-style: italic;
+        }
+
+        .available-tests-list {
+            list-style: none;
+            padding: 0;
+            margin: 0;
+        }
+
+        .available-tests-list li {
+            padding: 10px;
+            border-bottom: 1px solid var(--hairline-color, #eee);
+            list-style-type: none;
+        }
+
+        .available-tests-list li:last-child {
+            border-bottom: none;
+        }
+
+        .test-description {
+            color: var(--body-quiet-color);
+            font-size: 13px;
+            display: block;
+            margin-top: 4px;
+        }
+    </style>
+{% endblock %}
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+    <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+    &rsaquo; {% trans 'System Self-Tests' %}
+</div>
+{% endblock %}
+
+{% block content %}
+<div class="selftest-container">
+    {% if not run_tests %}
+        <a href="?run=true" class="run-tests-btn">{% trans "Run All Tests" %}</a>
+    {% else %}
+        <a href="{% url 'admin:selftest' %}" class="run-tests-btn" style="opacity: 0.8;">{% trans "Reset" %}</a>
+    {% endif %}
+
+    {% if run_tests %}
+        {% if all_passed is not None %}
+        <div class="selftest-module">
+            <div class="overall-status {% if all_passed %}success{% else %}failure{% endif %}">
+                {% if all_passed %}
+                    ✅ {% trans "All tests passed successfully!" %}
+                {% else %}
+                    ❌ {% trans "Some tests failed. Please check the details below." %}
+                {% endif %}
+            </div>
+        </div>
+        {% endif %}
+
+        <div class="selftest-module">
+            <h2>{% trans "Test Results" %}</h2>
+            <div class="module-content">
+                {% for result in results %}
+                    <div class="test-item">
+                        <div class="test-header">
+                            <span class="test-name">{{ result.name }}</span>
+                            <span class="test-status {% if result.success %}success{% else %}failure{% endif %}">
+                                {% if result.success %}
+                                    ✓ {% trans "Pass" %}
+                                {% else %}
+                                    ✗ {% trans "Fail" %}
+                                {% endif %}
+                            </span>
+                        </div>
+                        <div class="test-body">
+                            <div class="test-message">
+                                {{ result.message }}
+                                {% if result.duration_ms %}
+                                    <span class="duration">({{ result.duration_ms|floatformat:2 }}ms)</span>
+                                {% endif %}
+                            </div>
+                            {% if result.details %}
+                                <div class="test-details">
+                                    <strong>{% trans "Details:" %}</strong>
+                                    <dl>
+                                        {% for key, value in result.details.items %}
+                                            <dt>{{ key }}:</dt>
+                                            <dd>{{ value }}</dd>
+                                        {% endfor %}
+                                    </dl>
+                                </div>
+                            {% endif %}
+                        </div>
+                    </div>
+                {% endfor %}
+            </div>
+        </div>
+    {% else %}
+        <div class="selftest-module">
+            <h2>{% trans "Available Tests" %}</h2>
+            <div class="module-content">
+                <p>{% trans "Click 'Run All Tests' to execute the following system health checks:" %}</p>
+                <ul class="available-tests-list">
+                    {% for test in available_tests %}
+                        <li>
+                            <strong>{{ test.name }}</strong>
+                            <span class="test-description">{{ test.description }}</span>
+                        </li>
+                    {% endfor %}
+                </ul>
+            </div>
+        </div>
+    {% endif %}
+</div>
+{% endblock %}
+

src/backend/core/tests/commands/test_create_search_pipeline.py

@@ -10,9 +10,9 @@ import pytest
 
 from core.services.opensearch import opensearch_client
 from core.tests.utils import (
-    delete_search_pipeline,
     enable_hybrid_search,
 )
+from core.utils import delete_search_pipeline
 
 
 @pytest.fixture(autouse=True)
@@ -25,7 +25,6 @@ def before_each():
 
 def test_create_search_pipeline(settings, caplog):
     """Test command create search pipeline"""
-    # create documents and index them with hybrid search disabled
 
     enable_hybrid_search(settings)
 
@@ -38,12 +37,11 @@ def test_create_search_pipeline(settings, caplog):
     )
 
     # calling get works without raising NotFoundError
-    opensearch_client().search_pipeline.get(settings.HYBRID_SEARCH_PIPELINE_ID)
+    opensearch_client().search_pipeline.get(id=settings.HYBRID_SEARCH_PIPELINE_ID)
 
 
 def test_create_search_pipeline_but_it_exists_already(settings, caplog):
     """Test command create search pipeline but it already exists"""
-    # create documents and index them with hybrid search disabled
 
     opensearch_client().transport.perform_request(
         method="PUT",
@@ -75,4 +73,4 @@ def test_create_search_pipeline_but_it_exists_already(settings, caplog):
     )
 
     # the pipeline is still here
-    opensearch_client().search_pipeline.get(settings.HYBRID_SEARCH_PIPELINE_ID)
+    opensearch_client().search_pipeline.get(id=settings.HYBRID_SEARCH_PIPELINE_ID)

src/backend/core/tests/commands/test_reindex_with_embedding.py

@@ -19,9 +19,12 @@ from core.models import get_opensearch_index_name
 from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
 from core.tests.mock import albert_embedding_response
 from core.tests.utils import (
+    enable_hybrid_search,
+)
+from core.utils import (
     bulk_create_documents,
     delete_search_pipeline,
-    enable_hybrid_search,
+    get_language_value,
     prepare_index,
 )
 
@@ -61,12 +64,12 @@ def test_reindex_with_embedding_command(settings):
     prepare_index(index_name, documents)
 
     # the index has not been embedded in the initial state
-    initial_index = opensearch_client_.search(
+    initial_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
         index=index_name, size=3, body={"query": {"match_all": {}}}
     )
     assert len(initial_index["hits"]["hits"]) == 3
     for embedded_hit in initial_index["hits"]["hits"]:
-        assert embedded_hit["_source"]["embedding"] == None
+        assert embedded_hit["_source"]["chunks"] is None
         assert embedded_hit["_source"]["embedding_model"] is None
 
     # enable hybrid search
@@ -82,7 +85,7 @@ def test_reindex_with_embedding_command(settings):
     call_command("reindex_with_embedding", SERVICE_NAME)
 
     opensearch_client_.indices.refresh(index=index_name)
-    embedded_index = opensearch_client_.search(
+    embedded_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
         index=index_name, size=3, body={"query": {"match_all": {}}}
     )
 
@@ -91,8 +94,9 @@ def test_reindex_with_embedding_command(settings):
     for embedded_hit in embedded_index["hits"]["hits"]:
         embedded_source = embedded_hit["_source"]
         # the index contains a embedding and embedding_model
+        assert len(embedded_source["chunks"]) == 1
         assert (
-            embedded_source["embedding"]
+            embedded_source["chunks"][0]["embedding"]
             == albert_embedding_response.response["data"][0]["embedding"]
         )
         assert embedded_source["embedding_model"] == settings.EMBEDDING_API_MODEL_NAME
@@ -104,8 +108,8 @@ def test_reindex_with_embedding_command(settings):
         ]
         assert len(initial_hits) == 1
         initial_source = initial_hits[0]["_source"]
-        assert initial_source["title"] == embedded_source["title"]
-        assert initial_source["content"] == embedded_source["content"]
+        assert initial_source["title.en"] == embedded_source["title.en"]
+        assert initial_source["content.en"] == embedded_source["content.en"]
         assert initial_source["created_at"] == embedded_source["created_at"]
         assert initial_source["users"] == embedded_source["users"]
 
@@ -156,14 +160,14 @@ def test_reindex_can_fail_and_restart(settings):
 
     # assert the index state
     opensearch_client_.indices.refresh(index=index_name)
-    embedded_index = opensearch_client_.search(
+    embedded_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
         index=index_name, size=3, body={"query": {"match_all": {}}}
     )
     # Should have 2 documents with embeddings, 1 without due to error
     embedded_count = 0
     not_embedded_count = 0
     for hit in embedded_index["hits"]["hits"]:
-        if hit["_source"].get("embedding"):
+        if hit["_source"].get("chunks"):
             embedded_count += 1
             assert (
                 hit["_source"]["embedding_model"] == settings.EMBEDDING_API_MODEL_NAME
@@ -189,14 +193,15 @@ def test_reindex_can_fail_and_restart(settings):
 
     # assert there is now 1 more success and 0 failures
     opensearch_client_.indices.refresh(index=index_name)
-    embedded_index = opensearch_client_.search(
+    embedded_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
         index=index_name, size=3, body={"query": {"match_all": {}}}
     )
     for hit in embedded_index["hits"]["hits"]:
-        assert (
-            hit["_source"]["embedding"]
-            == albert_embedding_response.response["data"][0]["embedding"]
-        )
+        for chunk in hit["_source"]["chunks"]:
+            assert (
+                chunk["embedding"]
+                == albert_embedding_response.response["data"][0]["embedding"]
+            )
         assert hit["_source"]["embedding_model"] == settings.EMBEDDING_API_MODEL_NAME
 
 
@@ -204,7 +209,7 @@ def test_reindex_can_fail_and_restart(settings):
 def test_reindex_preserves_concurrent_updates(settings):
     """
     Test that concurrent document updates don't get overwritten by reindexing.
-    This test simulates the fallowing scenario:
+    This test simulates the following scenario:
     • the hybrid search is disabled
     • documents are created and indexed without indexing
     • the hybrid search is enabled
@@ -226,10 +231,6 @@ def test_reindex_preserves_concurrent_updates(settings):
     enable_hybrid_search(settings)
 
     updated_title = "updated dog"
-    updated_embedding = [
-        1.0
-    ] * settings.EMBEDDING_DIMENSION  # dummy embedding to simulate concurrent update
-    # add a side_effect on the search to simulate a concurrent update
     patch(
         "core.services.opensearch.opensearch_client_.search",
         side_effect=opensearch_client_.update(
@@ -237,9 +238,7 @@ def test_reindex_preserves_concurrent_updates(settings):
             id=documents[1]["id"],
             body={
                 "doc": {
-                    "title": updated_title,
-                    "embedding": updated_embedding,
-                    "embedding_model": settings.EMBEDDING_API_MODEL_NAME,
+                    "title.en": updated_title,
                 }
             },
         ),
@@ -256,18 +255,19 @@ def test_reindex_preserves_concurrent_updates(settings):
     assert result["nb_failed_embedding"] == 0
 
     opensearch_client_.indices.refresh(index=index_name)
-    embedded_index = opensearch_client_.search(
+    embedded_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
         index=index_name, size=2, body={"query": {"match_all": {}}}
     )
     # Check that the latest update is preserved
-    dog_doc = [
+    updated_document = [
         hit
         for hit in embedded_index["hits"]["hits"]
-        if hit["_source"]["title"] == updated_title
+        if get_language_value(hit["_source"], "title") == updated_title
     ]
-    assert len(dog_doc) == 1
-    assert dog_doc[0]["_source"]["embedding"] == updated_embedding
-    assert dog_doc[0]["_source"]["embedding_model"] == settings.EMBEDDING_API_MODEL_NAME
+    assert len(updated_document) == 1
+    # Check it was not embedded
+    assert updated_document[0]["_source"]["chunks"] is None
+    assert updated_document[0]["_source"]["embedding_model"] is None
 
 
 def test_reindex_command_but_hybrid_search_is_disabled():
@@ -279,7 +279,7 @@ def test_reindex_command_but_hybrid_search_is_disabled():
 
 
 def test_reindex_command_but_index_does_not_exist(settings):
-    """Test the `reindex_with_embedding` command fails when the idex does not exist."""
+    """Test the `reindex_with_embedding` command fails when the index does not exist."""
     wrong_index = "wrong-index-name"
     enable_hybrid_search(settings)
 

src/backend/core/tests/conftest.py

@@ -5,6 +5,7 @@ from faker import Faker
 from lasuite.oidc_resource_server.authentication import (
     get_resource_server_backend,
 )
+from opensearchpy.exceptions import NotFoundError
 
 from core.services import opensearch
 
@@ -47,5 +48,5 @@ def cleanup_test_index(settings):
 
     try:
         client.indices.delete(index=f"{prefix}-*")
-    except opensearch.NotFoundError:
+    except NotFoundError:
         pass

src/backend/core/tests/mock/albert_embedding_response.py

@@ -1,3 +1,7 @@
+"""Mock response for Albert embedding API."""
+
+# pylint: disable=too-many-lines
+
 response = {
     "data": [
         {

src/backend/core/tests/test_admin_selftest.py

@@ -0,0 +1,181 @@
+"""Tests for the admin selftest view."""
+
+from unittest.mock import patch
+
+from django.contrib.auth import get_user_model
+from django.urls import reverse
+
+import pytest
+
+from core.selftests import SelfTestResult
+
+pytestmark = pytest.mark.django_db
+
+User = get_user_model()
+
+
+@pytest.fixture(autouse=True)
+def _override_storage_settings(settings):
+    """Override storage settings for all tests."""
+    settings.STORAGES = {
+        "default": {
+            "BACKEND": "django.core.files.storage.FileSystemStorage",
+        },
+        "staticfiles": {
+            "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+        },
+    }
+
+
+def test_selftest_requires_authentication(client):
+    """Test that the selftest page requires authentication."""
+    selftest_url = reverse("admin:selftest")
+
+    response = client.get(selftest_url)
+
+    # Should redirect to login
+    assert response.status_code == 302
+    assert "/admin/login/" in response.url
+
+
+def test_selftest_requires_staff_permission(client):
+    """Test that only staff users can access the selftest page."""
+    selftest_url = reverse("admin:selftest")
+
+    User.objects.create_user(
+        username="user",
+        email="user@example.com",
+        password="user123",
+    )
+    client.login(username="user", password="user123")
+
+    response = client.get(selftest_url)
+
+    # Regular users should be redirected
+    assert response.status_code == 302
+
+
+def test_selftest_accessible_by_admin(client):
+    """Test that admin users can access the selftest page."""
+    selftest_url = reverse("admin:selftest")
+
+    User.objects.create_superuser(
+        username="admin",
+        email="admin@example.com",
+        password="admin123",
+    )
+    client.login(username="admin", password="admin123")
+
+    response = client.get(selftest_url)
+
+    assert response.status_code == 200
+    assert b"System Self-Tests" in response.content
+
+
+def test_selftest_displays_available_tests(client):
+    """Test that available tests are displayed when not running."""
+    selftest_url = reverse("admin:selftest")
+
+    User.objects.create_superuser(
+        username="admin",
+        email="admin@example.com",
+        password="admin123",
+    )
+    client.login(username="admin", password="admin123")
+
+    response = client.get(selftest_url)
+
+    assert response.status_code == 200
+    assert b"Available Tests" in response.content
+    assert b"Run All Tests" in response.content
+
+
+@patch("core.selftests.registry.run_all")
+def test_selftest_runs_tests(mock_run_all, client):
+    """Test that tests are executed when run=true."""
+    selftest_url = reverse("admin:selftest")
+
+    # Mock the test results
+    mock_run_all.return_value = [
+        SelfTestResult(
+            name="Test 1",
+            success=True,
+            message="Success",
+            duration_ms=10.0,
+        ),
+        SelfTestResult(
+            name="Test 2",
+            success=False,
+            message="Failed",
+            duration_ms=20.0,
+        ),
+    ]
+
+    User.objects.create_superuser(
+        username="admin",
+        email="admin@example.com",
+        password="admin123",
+    )
+    client.login(username="admin", password="admin123")
+
+    response = client.get(selftest_url, {"run": "true"})
+
+    assert response.status_code == 200
+    assert b"Test Results" in response.content
+    assert b"Test 1" in response.content
+    assert b"Test 2" in response.content
+    mock_run_all.assert_called_once()
+
+
+@patch("core.selftests.registry.run_all")
+def test_selftest_displays_success_status(mock_run_all, client):
+    """Test that success status is displayed correctly."""
+    selftest_url = reverse("admin:selftest")
+
+    mock_run_all.return_value = [
+        SelfTestResult(
+            name="Test 1",
+            success=True,
+            message="Success",
+            duration_ms=10.0,
+        ),
+    ]
+
+    User.objects.create_superuser(
+        username="admin",
+        email="admin@example.com",
+        password="admin123",
+    )
+    client.login(username="admin", password="admin123")
+
+    response = client.get(selftest_url, {"run": "true"})
+
+    assert response.status_code == 200
+    assert b"All tests passed successfully" in response.content
+
+
+@patch("core.selftests.registry.run_all")
+def test_selftest_displays_failure_status(mock_run_all, client):
+    """Test that failure status is displayed correctly."""
+    selftest_url = reverse("admin:selftest")
+
+    mock_run_all.return_value = [
+        SelfTestResult(
+            name="Test 1",
+            success=False,
+            message="Failed",
+            duration_ms=10.0,
+        ),
+    ]
+
+    User.objects.create_superuser(
+        username="admin",
+        email="admin@example.com",
+        password="admin123",
+    )
+    client.login(username="admin", password="admin123")
+
+    response = client.get(selftest_url, {"run": "true"})
+
+    assert response.status_code == 200
+    assert b"Some tests failed" in response.content

src/backend/core/tests/test_api_documents_delete.py

@@ -0,0 +1,384 @@
+"""Tests for deleting documents from OpenSearch over the API"""
+
+import opensearchpy
+import pytest
+import responses
+from rest_framework.test import APIClient
+
+from core import factories
+from core.services.opensearch import opensearch_client
+from core.utils import prepare_index
+
+from .utils import build_authorization_bearer, setup_oicd_resource_server
+
+pytestmark = pytest.mark.django_db
+
+
+def test_api_documents_delete_anonymous():
+    """Anonymous requests should not be allowed to delete documents."""
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": "service-name", "document_ids": ["doc1"]},
+        format="json",
+    )
+
+    assert response.status_code == 401
+
+
+@responses.activate
+def test_api_documents_delete_wrong_service_name(settings):
+    """Requests with a wrong service name should return 400 Bad Request."""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": "wrong-service", "document_ids": ["0"]},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 400
+    assert response.json()["detail"] == "Invalid request."
+
+
+@responses.activate
+def test_api_documents_delete_success(settings):
+    """Authenticated users should be able to delete documents they have access to."""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+
+    service = factories.ServiceFactory()
+    # Create documents user has access to
+    documents = factories.DocumentSchemaFactory.build_batch(3, users=["user_sub"])
+    prepare_index(service.index_name, documents)
+    document_to_delete_ids = [doc["id"] for doc in documents[:2]]
+
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": service.name, "document_ids": document_to_delete_ids},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert response.json()["nb-deleted-documents"] == 2
+    assert response.json()["undeleted-document-ids"] == []
+
+    opensearch_client_ = opensearch_client()
+    for document in documents:
+        if document["id"] in document_to_delete_ids:
+            with pytest.raises(opensearchpy.exceptions.NotFoundError):
+                opensearch_client_.get(index=service.index_name, id=document["id"])
+        else:
+            doc = opensearch_client_.get(index=service.index_name, id=document["id"])
+            assert doc["found"]
+
+
+@responses.activate
+def test_api_documents_delete_no_access(settings):
+    """Users should not be able to delete documents they don't have access to."""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    service = factories.ServiceFactory()
+    # Create documents where user_sub does NOT have access
+    documents = factories.DocumentSchemaFactory.build_batch(2, users=["other_sub"])
+    prepare_index(service.index_name, documents)
+
+    document_ids = [doc["id"] for doc in documents]
+
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": service.name, "document_ids": document_ids},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert response.json()["nb-deleted-documents"] == 0
+    assert set(response.json()["undeleted-document-ids"]) == set(document_ids)
+
+    # Verify documents not deleted
+    opensearch_client_ = opensearch_client()
+    for doc_id in document_ids:
+        doc = opensearch_client_.get(index=service.index_name, id=doc_id)
+        assert doc["found"]
+
+
+@responses.activate
+def test_api_documents_delete_mixed_access(settings):
+    """Deleting a mix of owned and non-owned documents should only delete owned ones."""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+
+    service = factories.ServiceFactory()
+    # Create documents with different access
+    owned_documents = factories.DocumentSchemaFactory.build_batch(2, users=["user_sub"])
+    other_documents = factories.DocumentSchemaFactory.build_batch(
+        2, users=["other_user"]
+    )
+    prepare_index(service.index_name, owned_documents + other_documents)
+
+    owned_document_ids = [doc["id"] for doc in owned_documents]
+    other_document_ids = [doc["id"] for doc in other_documents]
+    non_existing_document_ids = ["non-existent-1", "non-existent-2"]
+
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {
+            "service": service.name,
+            "document_ids": owned_document_ids
+            + other_document_ids
+            + non_existing_document_ids,
+        },
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert response.json()["nb-deleted-documents"] == 2
+    assert set(response.json()["undeleted-document-ids"]) == set(
+        other_document_ids + non_existing_document_ids
+    )
+
+    # Verify only owned documents are deleted
+    opensearch_client_ = opensearch_client()
+    for document_id in owned_document_ids:
+        with pytest.raises(opensearchpy.exceptions.NotFoundError):
+            opensearch_client_.get(index=service.index_name, id=document_id)
+
+    for document_id in other_document_ids:
+        document = opensearch_client_.get(index=service.index_name, id=document_id)
+        assert document["found"]
+
+
+@responses.activate
+def test_api_documents_delete_invalid_params(settings):
+    """Requests with invalid parameters should return 400 Bad Request."""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    service = factories.ServiceFactory()
+
+    # Missing both document_ids and tags
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {
+            "service": service.name,
+        },
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 400
+    assert (
+        response.json()[0]["msg"]
+        == "Value error, At least one of 'document_ids' or 'tags' must be provided"
+    )
+
+    # Empty document_ids and no tags
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": service.name, "document_ids": []},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 400
+    assert (
+        response.json()[0]["msg"]
+        == "Value error, At least one of 'document_ids' or 'tags' must be provided"
+    )
+
+    # Both empty
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": service.name, "document_ids": [], "tags": []},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 400
+    assert (
+        response.json()[0]["msg"]
+        == "Value error, At least one of 'document_ids' or 'tags' must be provided"
+    )
+
+    # Missing service
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"document_ids": ["doc1"]},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 400
+
+
+@responses.activate
+def test_api_documents_delete_nonexistent_documents(settings):
+    """
+    Deleting non-existent documents should not raise an error
+    and return the list of undeleted ids.
+    """
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    service = factories.ServiceFactory()
+    # Create index but with no documents
+    prepare_index(service.index_name, [])
+
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": service.name, "document_ids": ["non-existent-id"]},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert response.json()["nb-deleted-documents"] == 0
+    assert response.json()["undeleted-document-ids"] == ["non-existent-id"]
+
+
+@responses.activate
+def test_api_documents_delete_by_single_tag(settings):
+    """Users should be able to delete documents by tags."""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    service = factories.ServiceFactory()
+
+    document_to_deletes = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["delete-tag", "keep-tag-1"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["delete-tag", "keep-tag-2"]
+        ),
+    ]
+    document_to_keep = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["keep-tag-1", "keep-tag-2"]
+        ),
+        factories.DocumentSchemaFactory.build(users=["user_sub"], tags=["keep-tag-1"]),
+        factories.DocumentSchemaFactory.build(
+            users=["other_user_sub"], tags=["delete-tag"]
+        ),
+    ]
+    prepare_index(service.index_name, document_to_deletes + document_to_keep)
+
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": service.name, "tags": ["delete-tag"]},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert response.json()["nb-deleted-documents"] == 2
+    assert response.json()["undeleted-document-ids"] == []
+
+    opensearch_client_ = opensearch_client()
+    for document in document_to_deletes:
+        with pytest.raises(opensearchpy.exceptions.NotFoundError):
+            opensearch_client_.get(index=service.index_name, id=document["id"])
+
+    for document in document_to_keep:
+        doc = opensearch_client_.get(index=service.index_name, id=document["id"])
+        assert doc["found"]
+
+
+@responses.activate
+def test_api_documents_delete_by_multiple_tags(settings):
+    """Users should be able to delete documents matching any of multiple tags."""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    service = factories.ServiceFactory()
+
+    document_to_deletes = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["delete-tag-1", "keep-tag-1"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["delete-tag-1", "delete-tag-2"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["delete-tag-2"]
+        ),
+    ]
+    document_to_keep = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["keep-tag-1", "keep-tag-2"]
+        ),
+        factories.DocumentSchemaFactory.build(users=["user_sub"], tags=["keep-tag-1"]),
+        factories.DocumentSchemaFactory.build(
+            users=["other_user_sub"], tags=["delete-tag-1"]
+        ),
+    ]
+    prepare_index(service.index_name, document_to_deletes + document_to_keep)
+
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {"service": service.name, "tags": ["delete-tag-1", "delete-tag-2"]},
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert response.json()["nb-deleted-documents"] == 3
+    assert response.json()["undeleted-document-ids"] == []
+
+    opensearch_client_ = opensearch_client()
+    for document in document_to_deletes:
+        with pytest.raises(opensearchpy.exceptions.NotFoundError):
+            opensearch_client_.get(index=service.index_name, id=document["id"])
+
+    for document in document_to_keep:
+        doc = opensearch_client_.get(index=service.index_name, id=document["id"])
+        assert doc["found"]
+
+
+@responses.activate
+def test_api_documents_delete_by_ids_and_tags(settings):
+    """Users should be able to delete documents by both IDs and tags (AND logic)."""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    service = factories.ServiceFactory()
+
+    document_delete_by_tag_and_id = factories.DocumentSchemaFactory.build(
+        users=["user_sub"], tags=["delete-tag"]
+    )
+    document_delete_by_tag_keep_by_id = factories.DocumentSchemaFactory.build(
+        users=["user_sub"], tags=["delete-tag"]
+    )
+    document_keep_by_tag_delete_by_id = factories.DocumentSchemaFactory.build(
+        users=["user_sub"]
+    )
+
+    prepare_index(
+        service.index_name,
+        [
+            document_delete_by_tag_and_id,
+            document_delete_by_tag_keep_by_id,
+            document_keep_by_tag_delete_by_id,
+        ],
+    )
+
+    response = APIClient().post(
+        "/api/v1.0/documents/delete/",
+        {
+            "service": service.name,
+            "document_ids": [
+                document_delete_by_tag_and_id["id"],
+                document_keep_by_tag_delete_by_id["id"],
+            ],
+            "tags": ["delete-tag"],
+        },
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert response.json()["nb-deleted-documents"] == 1
+    assert response.json()["undeleted-document-ids"] == [
+        document_keep_by_tag_delete_by_id["id"]
+    ]
+
+    opensearch_client_ = opensearch_client()
+    with pytest.raises(opensearchpy.exceptions.NotFoundError):
+        opensearch_client_.get(
+            index=service.index_name, id=document_delete_by_tag_and_id["id"]
+        )
+
+    doc = opensearch_client_.get(
+        index=service.index_name, id=document_delete_by_tag_keep_by_id["id"]
+    )
+    assert doc["found"]

src/backend/core/tests/test_api_documents_index_bulk.py

@@ -6,6 +6,7 @@ from unittest import mock
 from django.utils import timezone
 
 import pytest
+from opensearchpy import NotFoundError
 from rest_framework.test import APIClient
 
 from core import factories
@@ -59,12 +60,12 @@ def test_api_documents_index_bulk_success():
 
 
 def test_api_documents_index_bulk_ensure_index():
-    """A registered service should be create the opensearch index if need."""
+    """A registered service should be created the opensearch index if needed."""
     opensearch_client_ = opensearch.opensearch_client()
     service = factories.ServiceFactory()
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
-    with pytest.raises(opensearch.NotFoundError):
+    with pytest.raises(NotFoundError):
         opensearch_client_.indices.get(index=service.index_name)
 
     response = APIClient().post(

src/backend/core/tests/test_api_documents_index_single.py

@@ -1,11 +1,13 @@
 """Tests indexing documents in OpenSearch over the API"""
 
 import datetime
+import logging
 
 from django.utils import timezone
 
 import pytest
 import responses
+from opensearchpy import NotFoundError
 from rest_framework.test import APIClient
 
 from core import factories
@@ -53,8 +55,7 @@ def test_api_documents_index_single_invalid_token():
 def test_api_documents_index_single_hybrid_enabled_success(settings):
     """
     A registered service should be able to index document with a valid token.
-    If hybrid search is enabled, the indexing should have embedding of
-    dimension settings.EMBEDDING_DIMENSION.
+    If hybrid search is enabled, the documents are chunked and embedded.
     """
     service = factories.ServiceFactory()
     enable_hybrid_search(settings)
@@ -66,6 +67,9 @@ def test_api_documents_index_single_hybrid_enabled_success(settings):
     )
 
     document = factories.DocumentSchemaFactory.build()
+    document["content"] = (
+        "a long text to embed." * 100
+    )  # Ensure content is long enough for chunking
 
     response = APIClient().post(
         "/api/v1.0/documents/index/",
@@ -81,16 +85,158 @@ def test_api_documents_index_single_hybrid_enabled_success(settings):
         index=service.index_name, id=str(document["id"])
     )
     assert new_indexed_document["_version"] == 1
-    assert new_indexed_document["_source"]["title"] == document["title"].strip().lower()
-    assert new_indexed_document["_source"]["content"] == document["content"]
+
     assert (
-        new_indexed_document["_source"]["embedding"]
+        new_indexed_document["_source"]["title.en"] == document["title"].strip().lower()
+    )
+    assert new_indexed_document["_source"]["content.en"] == document["content"]
+    # only the english fields are indexed
+    assert not "content.fr" in new_indexed_document["_source"]
+
+    # check embedding
+    assert (
+        new_indexed_document["_source"]["chunks"][0]["embedding"]
         == albert_embedding_response.response["data"][0]["embedding"]
     )
     assert (
         new_indexed_document["_source"]["embedding_model"]
         == settings.EMBEDDING_API_MODEL_NAME
     )
+    # Check that the document has been chunked correctly
+    assert (
+        len(new_indexed_document["_source"]["chunks"])
+        == int(
+            len(document["content"]) / (settings.CHUNK_SIZE - settings.CHUNK_OVERLAP)
+        )
+        + 1
+    )
+    for chunk in new_indexed_document["_source"]["chunks"]:
+        assert (
+            chunk["embedding"]
+            == albert_embedding_response.response["data"][0]["embedding"]
+        )
+        assert chunk["content"] in document["content"]
+        assert len(chunk["content"]) < len(document["content"])
+
+
+@responses.activate
+def test_api_documents_index_with_wrong_embedding_dimension(settings, caplog):
+    """Test embedding with wrong dimension should log a warning and not index the embedding."""
+    service = factories.ServiceFactory()
+    enable_hybrid_search(settings)
+    settings.EMBEDDING_DIMENSION = 8
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+
+    document = factories.DocumentSchemaFactory.build()
+
+    with caplog.at_level(logging.WARNING):
+        APIClient().post(
+            "/api/v1.0/documents/index/",
+            document,
+            HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
+            format="json",
+        )
+
+    assert any(
+        "unexpected embedding dimension: EMBEDDING_DIMENSION is set to 8 "
+        "but the configured embedding model returned a vector of dimension 1024"
+        in message
+        for message in caplog.messages
+    )
+
+    new_indexed_document = opensearch.opensearch_client().get(
+        index=service.index_name, id=str(document["id"])
+    )
+    # check embedding
+    assert new_indexed_document["_source"]["chunks"] is None
+
+
+def test_api_documents_index_language_params():
+    """language_code query param should control which language is indexed."""
+    service = factories.ServiceFactory()
+    document = factories.DocumentSchemaFactory.build()
+
+    response = APIClient().post(
+        "/api/v1.0/documents/index/",
+        document,
+        HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
+        format="json",
+    )
+
+    assert response.status_code == 201
+    assert response.json()["_id"] == str(document["id"])
+
+    new_indexed_document = opensearch.opensearch_client().get(
+        index=service.index_name, id=str(document["id"])
+    )
+    language_code = "en"
+    assert (
+        new_indexed_document["_source"][f"title.{language_code}"]
+        == document["title"].strip().lower()
+    )
+    assert (
+        new_indexed_document["_source"][f"content.{language_code}"]
+        == document["content"]
+    )
+    other_language_code = "fr"
+    # only the requested language is indexed
+    assert not f"content.{other_language_code}" in new_indexed_document["_source"]
+
+
+def test_api_documents_index_and_reindex_same_document():
+    """
+    Indexing the same document twice should update it.
+    If the detected language changes the new language code should be used and the
+    former language code should not be present anymore.
+    """
+    service = factories.ServiceFactory()
+    document = factories.DocumentSchemaFactory.build()
+
+    # First indexing with unrecognized language title
+    document["title"] = "planning"
+    APIClient().post(
+        "/api/v1.0/documents/index/",
+        document,
+        HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
+        format="json",
+    )
+
+    new_indexed_document = opensearch.opensearch_client().get(
+        index=service.index_name, id=str(document["id"])
+    )
+    assert new_indexed_document["_version"] == 1
+    assert (
+        new_indexed_document["_source"]["title.und"]
+        == document["title"].strip().lower()
+    )
+    assert new_indexed_document["_source"]["content.und"] == document["content"].strip()
+
+    # Index the same document with a french content
+    document["content"] = "du contenu en francais"
+    APIClient().post(
+        "/api/v1.0/documents/index/",
+        document,
+        HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
+        format="json",
+    )
+
+    new_indexed_document = opensearch.opensearch_client().get(
+        index=service.index_name, id=str(document["id"])
+    )
+    assert new_indexed_document["_version"] == 2
+    # the document is detected as french
+    assert (
+        new_indexed_document["_source"]["title.fr"] == document["title"].strip().lower()
+    )
+    assert new_indexed_document["_source"]["content.fr"] == document["content"]
+    # und field are removed
+    assert "title.und" not in new_indexed_document["_source"]
+    assert "content.und" not in new_indexed_document["_source"]
 
 
 def test_api_documents_index_single_hybrid_disabled_success():
@@ -113,18 +259,20 @@ def test_api_documents_index_single_hybrid_disabled_success():
         index=service.index_name, id=str(document["id"])
     )
     assert new_indexed_document["_version"] == 1
-    assert new_indexed_document["_source"]["title"] == document["title"].strip().lower()
-    assert new_indexed_document["_source"]["content"] == document["content"]
-    assert new_indexed_document["_source"]["embedding"] is None
+    assert (
+        new_indexed_document["_source"]["title.en"] == document["title"].strip().lower()
+    )
+    assert new_indexed_document["_source"]["content.en"] == document["content"]
+    assert new_indexed_document["_source"]["chunks"] is None
 
 
 def test_api_documents_index_single_ensure_index(settings):
-    """A registered service should be create the opensearch index if need."""
+    """A registered service should be created the opensearch index if needed."""
     service = factories.ServiceFactory()
     document = factories.DocumentSchemaFactory.build()
     opensearch_client_ = opensearch.opensearch_client()
 
-    with pytest.raises(opensearch.NotFoundError):
+    with pytest.raises(NotFoundError):
         opensearch_client_.indices.get(index=service.index_name)
 
     response = APIClient().post(
@@ -143,38 +291,154 @@ def test_api_documents_index_single_ensure_index(settings):
     assert data[service.index_name]["mappings"] == {
         "dynamic": "strict",
         "properties": {
-            "id": {"type": "keyword"},
-            "title": {
-                "type": "keyword",  # Primary field for exact matches and sorting
-                "fields": {
-                    "text": {"type": "text"}  # Sub-field for full-text search
+            "chunks": {
+                "type": "nested",
+                "properties": {
+                    "content": {"type": "text"},
+                    "embedding": {
+                        "type": "knn_vector",
+                        "dimension": settings.EMBEDDING_DIMENSION,
+                        "method": {
+                            "engine": "lucene",
+                            "space_type": "l2",
+                            "name": "hnsw",
+                            "parameters": {},
+                        },
+                    },
+                    "index": {"type": "integer"},
                 },
             },
-            "depth": {"type": "integer"},
-            "path": {
-                "type": "keyword",
-                "fields": {"text": {"type": "text"}},
+            "content": {
+                "properties": {
+                    "de": {
+                        "type": "text",
+                        "fields": {
+                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
+                        },
+                        "analyzer": "german_analyzer",
+                    },
+                    "en": {
+                        "type": "text",
+                        "fields": {
+                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
+                        },
+                        "analyzer": "english_analyzer",
+                    },
+                    "fr": {
+                        "type": "text",
+                        "fields": {
+                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
+                        },
+                        "analyzer": "french_analyzer",
+                    },
+                    "nl": {
+                        "type": "text",
+                        "fields": {
+                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
+                        },
+                        "analyzer": "dutch_analyzer",
+                    },
+                    "und": {
+                        "type": "text",
+                        "fields": {
+                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
+                        },
+                        "analyzer": "undetermined_language_analyzer",
+                    },
+                }
             },
-            "numchild": {"type": "integer"},
-            "content": {"type": "text"},
             "created_at": {"type": "date"},
-            "updated_at": {"type": "date"},
-            "size": {"type": "long"},
-            "users": {"type": "keyword"},
-            "groups": {"type": "keyword"},
-            "reach": {"type": "keyword"},
-            "is_active": {"type": "boolean"},
-            "embedding": {
-                "type": "knn_vector",
-                "dimension": settings.EMBEDDING_DIMENSION,
-                "method": {
-                    "engine": "lucene",
-                    "space_type": "l2",
-                    "name": "hnsw",
-                    "parameters": {},
-                },
-            },
+            "depth": {"type": "integer"},
             "embedding_model": {"type": "keyword"},
+            "groups": {"type": "keyword"},
+            "id": {"type": "keyword"},
+            "is_active": {"type": "boolean"},
+            "numchild": {"type": "integer"},
+            "path": {"type": "keyword", "fields": {"text": {"type": "text"}}},
+            "reach": {"type": "keyword"},
+            "size": {"type": "long"},
+            "tags": {"type": "keyword"},
+            "title": {
+                "properties": {
+                    "de": {
+                        "type": "keyword",
+                        "fields": {
+                            "text": {
+                                "type": "text",
+                                "fields": {
+                                    "trigrams": {
+                                        "type": "text",
+                                        "analyzer": "trigram_analyzer",
+                                    }
+                                },
+                                "analyzer": "german_analyzer",
+                            }
+                        },
+                    },
+                    "en": {
+                        "type": "keyword",
+                        "fields": {
+                            "text": {
+                                "type": "text",
+                                "fields": {
+                                    "trigrams": {
+                                        "type": "text",
+                                        "analyzer": "trigram_analyzer",
+                                    }
+                                },
+                                "analyzer": "english_analyzer",
+                            }
+                        },
+                    },
+                    "fr": {
+                        "type": "keyword",
+                        "fields": {
+                            "text": {
+                                "type": "text",
+                                "fields": {
+                                    "trigrams": {
+                                        "type": "text",
+                                        "analyzer": "trigram_analyzer",
+                                    }
+                                },
+                                "analyzer": "french_analyzer",
+                            }
+                        },
+                    },
+                    "nl": {
+                        "type": "keyword",
+                        "fields": {
+                            "text": {
+                                "type": "text",
+                                "fields": {
+                                    "trigrams": {
+                                        "type": "text",
+                                        "analyzer": "trigram_analyzer",
+                                    }
+                                },
+                                "analyzer": "dutch_analyzer",
+                            }
+                        },
+                    },
+                    "und": {
+                        "type": "keyword",
+                        "fields": {
+                            "text": {
+                                "type": "text",
+                                "fields": {
+                                    "trigrams": {
+                                        "type": "text",
+                                        "analyzer": "trigram_analyzer",
+                                    }
+                                },
+                                "analyzer": "undetermined_language_analyzer",
+                            }
+                        },
+                    },
+                }
+            },
+            "updated_at": {"type": "date"},
+            "users": {"type": "keyword"},
         },
     }
 

src/backend/core/tests/test_api_documents_search.py

@@ -5,7 +5,7 @@ Don't use pytest parametrized tests because batch generation and indexing
 of documents is slow and better done only once.
 """
 
-import operator
+import logging
 import random
 
 import pytest
@@ -13,14 +13,17 @@ import responses
 from rest_framework.test import APIClient
 
 from core import enums, factories
-from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
+from core.services.opensearch import (
+    check_hybrid_search_enabled,
+    opensearch_client,
+)
+from core.utils import bulk_create_documents, prepare_index
 
+from ..enums import SearchTypeEnum
 from .mock import albert_embedding_response
 from .utils import (
     build_authorization_bearer,
-    bulk_create_documents,
     enable_hybrid_search,
-    prepare_index,
     setup_oicd_resource_server,
 )
 
@@ -98,8 +101,6 @@ def test_api_documents_search_query_unknown_user(settings):
         introspect=lambda request, user_info: (404, {}, ""),
     )
 
-    token = build_authorization_bearer()
-
     service = factories.ServiceFactory()
     prepare_index(service.index_name, [])
 
@@ -107,11 +108,10 @@ def test_api_documents_search_query_unknown_user(settings):
         "/api/v1.0/documents/search/",
         {"q": "a quick fox"},
         format="json",
-        HTTP_AUTHORIZATION=f"Bearer {token}",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
 
-    assert response.status_code == 401
-    assert response.json() == {"detail": "Login failed"}
+    assert response.status_code == 400
 
 
 @responses.activate
@@ -178,7 +178,6 @@ def test_api_documents_search_reached_docs_invalid_parameters(settings):
 def test_api_documents_search_match_all(settings):
     """Searching a document with q='*' should match all docs"""
     setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
     responses.add(
         responses.POST,
         settings.EMBEDDING_API_PATH,
@@ -194,9 +193,13 @@ def test_api_documents_search_match_all(settings):
 
     response = APIClient().post(
         "/api/v1.0/documents/search/",
-        {"q": "*", "visited": [doc["id"] for doc in documents]},
+        {
+            "q": "*",
+            "visited": [doc["id"] for doc in documents],
+            "rescore": False,
+        },
         format="json",
-        HTTP_AUTHORIZATION=f"Bearer {token}",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
 
     assert response.status_code == 200
@@ -236,6 +239,7 @@ def test_api_documents_full_text_search_query_title(settings):
     assert list(fox_response.keys()) == ["_index", "_id", "_score", "_source", "fields"]
     assert fox_response["_id"] == str(documents[0]["id"])
     assert fox_response["_source"] == {
+        "content.en": "the wolf",
         "depth": 1,
         "numchild": 0,
         "path": fox_document["path"],
@@ -243,7 +247,8 @@ def test_api_documents_full_text_search_query_title(settings):
         "created_at": fox_document["created_at"].isoformat(),
         "updated_at": fox_document["updated_at"].isoformat(),
         "reach": fox_document["reach"],
-        "title": fox_document["title"],
+        "tags": [],
+        "title.en": fox_document["title"],
     }
     assert fox_response["fields"] == {"number_of_users": [1], "number_of_groups": [3]}
 
@@ -258,6 +263,7 @@ def test_api_documents_full_text_search_query_title(settings):
     ]
     assert other_fox_response["_id"] == str(other_fox_document["id"])
     assert other_fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": other_fox_document["path"],
@@ -265,7 +271,8 @@ def test_api_documents_full_text_search_query_title(settings):
         "created_at": other_fox_document["created_at"].isoformat(),
         "updated_at": other_fox_document["updated_at"].isoformat(),
         "reach": other_fox_document["reach"],
-        "title": other_fox_document["title"],
+        "tags": [],
+        "title.en": other_fox_document["title"],
     }
     assert other_fox_response["fields"] == {
         "number_of_users": [1],
@@ -275,9 +282,10 @@ def test_api_documents_full_text_search_query_title(settings):
 
 @responses.activate
 def test_api_documents_full_text_search(settings):
-    """Searching a document by its content should work as expected"""
+    """
+    Searching a document by its content should work as expected.
+    """
     setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
 
     service = factories.ServiceFactory()
     documents = bulk_create_documents(
@@ -293,7 +301,7 @@ def test_api_documents_full_text_search(settings):
         "/api/v1.0/documents/search/",
         {"q": "a quick fox", "visited": [doc["id"] for doc in documents]},
         format="json",
-        HTTP_AUTHORIZATION=f"Bearer {token}",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
 
     assert response.status_code == 200
@@ -305,6 +313,7 @@ def test_api_documents_full_text_search(settings):
     assert fox_response["_id"] == str(fox_document["id"])
     assert fox_response["_score"] > 0
     assert fox_response["_source"] == {
+        "content.en": "the wolf",
         "depth": 1,
         "numchild": 0,
         "path": fox_document["path"],
@@ -312,7 +321,8 @@ def test_api_documents_full_text_search(settings):
         "created_at": fox_document["created_at"].isoformat(),
         "updated_at": fox_document["updated_at"].isoformat(),
         "reach": fox_document["reach"],
-        "title": fox_document["title"],
+        "tags": [],
+        "title.en": fox_document["title"],
     }
     assert fox_response["fields"] == {"number_of_users": [1], "number_of_groups": [3]}
 
@@ -328,6 +338,7 @@ def test_api_documents_full_text_search(settings):
     assert other_fox_response["_id"] == str(other_fox_document["id"])
     assert other_fox_response["_score"] > 0
     assert other_fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": other_fox_document["path"],
@@ -335,7 +346,8 @@ def test_api_documents_full_text_search(settings):
         "created_at": other_fox_document["created_at"].isoformat(),
         "updated_at": other_fox_document["updated_at"].isoformat(),
         "reach": other_fox_document["reach"],
-        "title": other_fox_document["title"],
+        "tags": [],
+        "title.en": other_fox_document["title"],
     }
     assert other_fox_response["fields"] == {
         "number_of_users": [1],
@@ -343,11 +355,47 @@ def test_api_documents_full_text_search(settings):
     }
 
 
+@responses.activate
+def test_api_documents_search_with_search_type_full_text(settings, caplog):
+    """Test API with search_type=full_text forces full-text search even if hybrid is enabled"""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    service = factories.ServiceFactory()
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+        ]
+    )
+    prepare_index(service.index_name, documents)
+    with caplog.at_level(logging.INFO):
+        response = APIClient().post(
+            "/api/v1.0/documents/search/",
+            {
+                "q": "wolf",
+                "search_type": SearchTypeEnum.FULL_TEXT,
+                "visited": [doc["id"] for doc in documents],
+            },
+            format="json",
+            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+        )
+    assert response.status_code == 200
+    assert any(
+        "Performing full-text search without embedding: wolf" in message
+        for message in caplog.messages
+    )
+
+
 @responses.activate
 def test_api_documents_hybrid_search(settings):
     """Searching a document by its content should work as expected"""
     setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
     # hybrid search is enabled by default
     enable_hybrid_search(settings)
     responses.add(
@@ -371,7 +419,7 @@ def test_api_documents_hybrid_search(settings):
         "/api/v1.0/documents/search/",
         {"q": "a quick fox", "visited": [doc["id"] for doc in documents]},
         format="json",
-        HTTP_AUTHORIZATION=f"Bearer {token}",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
 
     assert response.status_code == 200
@@ -385,6 +433,7 @@ def test_api_documents_hybrid_search(settings):
     assert fox_response["_id"] == str(fox_document["id"])
     assert fox_response["_score"] > 0
     assert fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": fox_document["path"],
@@ -392,7 +441,8 @@ def test_api_documents_hybrid_search(settings):
         "created_at": fox_document["created_at"].isoformat(),
         "updated_at": fox_document["updated_at"].isoformat(),
         "reach": fox_document["reach"],
-        "title": fox_document["title"],
+        "tags": [],
+        "title.en": fox_document["title"],
     }
     assert fox_response["fields"] == {"number_of_users": [1], "number_of_groups": [3]}
 
@@ -408,6 +458,7 @@ def test_api_documents_hybrid_search(settings):
     assert other_fox_response["_id"] == str(other_fox_document["id"])
     assert other_fox_response["_score"] > 0
     assert other_fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": other_fox_document["path"],
@@ -415,7 +466,8 @@ def test_api_documents_hybrid_search(settings):
         "created_at": other_fox_document["created_at"].isoformat(),
         "updated_at": other_fox_document["updated_at"].isoformat(),
         "reach": other_fox_document["reach"],
-        "title": other_fox_document["title"],
+        "tags": [],
+        "title.en": other_fox_document["title"],
     }
     assert other_fox_response["fields"] == {
         "number_of_users": [1],
@@ -433,6 +485,7 @@ def test_api_documents_hybrid_search(settings):
     ]
     assert no_fox_response["_id"] == str(no_fox_document["id"])
     assert no_fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": no_fox_document["path"],
@@ -440,7 +493,8 @@ def test_api_documents_hybrid_search(settings):
         "created_at": no_fox_document["created_at"].isoformat(),
         "updated_at": no_fox_document["updated_at"].isoformat(),
         "reach": no_fox_document["reach"],
-        "title": no_fox_document["title"],
+        "tags": [],
+        "title.en": no_fox_document["title"],
     }
     assert no_fox_response["fields"] == {
         "number_of_users": [1],
@@ -448,192 +502,10 @@ def test_api_documents_hybrid_search(settings):
     }
 
 
-@responses.activate
-def test_api_documents_search_ordering_by_fields(settings):
-    """It should be possible to order by several fields"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        4, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    parameters = [
-        (enums.TITLE, "asc"),
-        (enums.TITLE, "desc"),
-        (enums.CREATED_AT, "asc"),
-        (enums.CREATED_AT, "desc"),
-        (enums.UPDATED_AT, "asc"),
-        (enums.UPDATED_AT, "desc"),
-        (enums.SIZE, "asc"),
-        (enums.SIZE, "desc"),
-        (enums.REACH, "asc"),
-        (enums.REACH, "desc"),
-    ]
-
-    for field, direction in parameters:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": field,
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {token}",
-        )
-
-        assert response.status_code == 200
-        data = response.json()
-        assert len(data) == 4
-
-        # Check that results are sorted by the field as expected
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(data) - 1):
-            assert compare(data[i]["_source"][field], data[i + 1]["_source"][field])
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_relevance(settings):
-    """It should be possible to order by relevance (score)"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        4, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    for direction in ["asc", "desc"]:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": "relevance",
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {token}",
-        )
-
-        assert response.status_code == 200
-        data = response.json()
-        assert len(data) == 4
-
-        # Check that results are sorted by score as expected
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(data) - 1):
-            assert compare(data[i]["_score"], data[i + 1]["_score"])
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_unknown_field(settings):
-    """Trying to sort by an unknown field should return a 400 error"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    # Setup: Initialize the service and documents only once
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        2, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    # Define the parameters manually
-    directions = ["asc", "desc"]
-
-    # Perform the parameterized tests
-    for direction in directions:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": "unknown",
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {token}",
-        )
-
-        assert response.status_code == 400
-        assert response.json() == [
-            {
-                "loc": ["order_by"],
-                "msg": (
-                    "Input should be 'relevance', 'title', 'created_at', "
-                    "'updated_at', 'size' or 'reach'"
-                ),
-                "type": "literal_error",
-            }
-        ]
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_unknown_direction(settings):
-    """Trying to sort with an unknown direction should return a 400 error"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        2, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    for field in enums.ORDER_BY_OPTIONS:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": field,
-                "order_direction": "unknown",
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {token}",
-        )
-
-        assert response.status_code == 400
-        assert response.json() == [
-            {
-                "loc": ["order_direction"],
-                "msg": "Input should be 'asc' or 'desc'",
-                "type": "literal_error",
-            }
-        ]
-
-
 @responses.activate
 def test_api_documents_search_filtering_by_reach(settings):
     """It should be possible to filter results by their reach"""
     setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
     responses.add(
         responses.POST,
         settings.EMBEDDING_API_PATH,
@@ -655,7 +527,7 @@ def test_api_documents_search_filtering_by_reach(settings):
                 "visited": [doc["id"] for doc in documents],
             },
             format="json",
-            HTTP_AUTHORIZATION=f"Bearer {token}",
+            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
         )
 
         assert response.status_code == 200
@@ -669,7 +541,6 @@ def test_api_documents_search_filtering_by_reach(settings):
 def test_api_documents_search_with_nb_results(settings):
     """nb_size should correctly return results of given size"""
     setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
     responses.add(
         responses.POST,
         settings.EMBEDDING_API_PATH,
@@ -690,9 +561,10 @@ def test_api_documents_search_with_nb_results(settings):
             "q": "*",
             "nb_results": nb_results,
             "visited": [doc["id"] for doc in documents],
+            "rescore": False,
         },
         format="json",
-        HTTP_AUTHORIZATION=f"Bearer {token}",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
 
     assert response.status_code == 200
@@ -708,11 +580,11 @@ def test_api_documents_search_with_nb_results(settings):
             "visited": [doc["id"] for doc in documents],
         },
         format="json",
-        HTTP_AUTHORIZATION=f"Bearer {token}",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
     assert response.status_code == 200
     data = response.json()
-    assert [r["_id"] for r in data] == ids[0:nb_results]
+    assert {r["_id"] for r in data} == set(ids[0:nb_results])
 
     nb_results = 10
     response = APIClient().post(
@@ -723,19 +595,18 @@ def test_api_documents_search_with_nb_results(settings):
             "visited": [doc["id"] for doc in documents],
         },
         format="json",
-        HTTP_AUTHORIZATION=f"Bearer {token}",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
     assert response.status_code == 200
     data = response.json()
     # nb_results > total number of documents => returns all documents
-    assert [r["_id"] for r in data] == ids[0:9]
+    assert {r["_id"] for r in data} == set(ids[0:9])
 
 
 @responses.activate
 def test_api_documents_search_nb_results_invalid_parameters(settings):
     """Invalid nb_results parameters should result in a 400 error"""
     setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
     responses.add(
         responses.POST,
         settings.EMBEDDING_API_PATH,
@@ -764,7 +635,7 @@ def test_api_documents_search_nb_results_invalid_parameters(settings):
             "/api/v1.0/documents/search/",
             {"q": "*", "nb_results": nb_results},
             format="json",
-            HTTP_AUTHORIZATION=f"Bearer {token}",
+            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
         )
 
         assert response.status_code == 400
@@ -776,7 +647,6 @@ def test_api_documents_search_nb_results_invalid_parameters(settings):
 def test_api_documents_search_nb_results_with_filtering(settings):
     """nb_results should work correctly when combined with filtering by reach"""
     setup_oicd_resource_server(responses, settings, sub="user_sub")
-    token = build_authorization_bearer()
     responses.add(
         responses.POST,
         settings.EMBEDDING_API_PATH,
@@ -799,9 +669,157 @@ def test_api_documents_search_nb_results_with_filtering(settings):
             "reach": "public",
             "nb_results": nb_results,
             "visited": public_ids,
+            "rescore_enable": False,
         },
         format="json",
-        HTTP_AUTHORIZATION=f"Bearer {token}",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
     assert response.status_code == 200
-    assert [r["_id"] for r in response.json()] == public_ids[0:nb_results]
+    assert {r["_id"] for r in response.json()} == set(public_ids[0:nb_results])
+
+
+@responses.activate
+def test_api_documents_search_filtering_by_tags(settings):
+    """Test filtering documents by a single tag via API"""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    service = factories.ServiceFactory()
+
+    documents = bulk_create_documents(
+        [
+            {
+                "title": "Python document",
+                "content": "About Python",
+                "tags": ["python", "programming"],
+            },
+            {
+                "title": "JavaScript document",
+                "content": "About JavaScript",
+                "tags": ["javascript", "programming"],
+            },
+            {
+                "title": "Untagged document",
+                "content": "No tags here",
+                "tags": [],
+            },
+        ]
+    )
+
+    prepare_index(service.index_name, documents)
+
+    response = APIClient().post(
+        "/api/v1.0/documents/search/",
+        {
+            "q": "*",
+            "tags": ["python"],
+            "visited": [doc["id"] for doc in documents],
+        },
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert len(response.json()) == 1
+    assert response.json()[0]["_id"] == str(documents[0]["id"])
+    assert response.json()[0]["_source"]["tags"] == ["python", "programming"]
+
+
+@responses.activate
+def test_api_documents_search_without_tags_filter(settings):
+    """Test that search works normally when no tags filter is provided"""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    service = factories.ServiceFactory()
+
+    documents = bulk_create_documents(
+        [
+            {
+                "title": "Document with tags",
+                "content": "Tagged content",
+                "tags": ["python"],
+            },
+            {
+                "title": "Document without tags",
+                "content": "Untagged content",
+                "tags": [],
+            },
+        ]
+    )
+
+    prepare_index(service.index_name, documents)
+
+    # Search without tags parameter - should return all documents
+    response = APIClient().post(
+        "/api/v1.0/documents/search/",
+        {
+            "q": "*",
+            "visited": [doc["id"] for doc in documents],
+        },
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert len(response.json()) == 2
+
+
+@responses.activate
+def test_api_documents_search_filtering_by_path(settings):
+    """Test filtering documents by path prefix via API"""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    service = factories.ServiceFactory()
+
+    documents = bulk_create_documents(
+        [
+            {
+                "title": "Document with tags",
+                "content": "Tagged document",
+                "path": "/path/to/doc1",
+            },
+            {
+                "title": "Document without tags",
+                "content": "Untagged document",
+                "path": "/path/to/doc2",
+            },
+            {
+                "title": "Document without tags",
+                "content": "Untagged document",
+                "path": "other/path/to/doc3",
+            },
+        ]
+    )
+
+    prepare_index(service.index_name, documents)
+
+    path_filter = "/path/to/"
+    response = APIClient().post(
+        "/api/v1.0/documents/search/",
+        {
+            "q": "*",
+            "path": path_filter,
+            "visited": [doc["id"] for doc in documents],
+        },
+        format="json",
+        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+    )
+
+    assert response.status_code == 200
+    assert len(response.json()) == 2
+    for hit in response.json():
+        assert hit["_source"]["path"].startswith(path_filter)

src/backend/core/tests/test_api_documents_search_access_control.py

@@ -11,11 +11,11 @@ from rest_framework.test import APIClient
 
 from core import enums, factories
 from core.services.opensearch import opensearch_client
+from core.utils import prepare_index
 
 from .mock import albert_embedding_response
 from .utils import (
     build_authorization_bearer,
-    prepare_index,
     setup_oicd_resource_server,
 )
 
@@ -395,7 +395,7 @@ def test_api_documents_search_access__request_services(settings):
     )
 
     assert response.status_code == 400
-    assert response.json() == {"detail": "Some requested services are not available"}
+    assert response.json() == {"detail": "Invalid request."}
 
 
 @responses.activate
@@ -419,7 +419,7 @@ def test_api_documents_search_access__request_inactive_services(settings):
     )
 
     assert response.status_code == 400
-    assert response.json() == {"detail": "Service is not available"}
+    assert response.json() == {"detail": "Invalid request."}
 
     # Event without explicit argument, the client service from the request is not active
     response = APIClient().post(
@@ -430,7 +430,7 @@ def test_api_documents_search_access__request_inactive_services(settings):
     )
 
     assert response.status_code == 400
-    assert response.json() == {"detail": "Service is not available"}
+    assert response.json() == {"detail": "Invalid request."}
 
 
 @responses.activate

src/backend/core/tests/test_embedding.py

@@ -0,0 +1,125 @@
+"""
+Test suite for opensearch embedding service
+"""
+
+import logging
+from json import dumps as json_dumps
+
+import pytest
+import responses
+
+from core.services.embedding import embed_text
+from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
+
+from .mock import albert_embedding_response
+from .utils import (
+    check_hybrid_search_enabled as check_hybrid_search_enabled_utils,
+)
+from .utils import (
+    enable_hybrid_search,
+)
+
+pytestmark = pytest.mark.django_db
+
+
+@pytest.fixture(autouse=True)
+def before_each():
+    """Clear caches before each test"""
+    clear_caches()
+    yield
+    clear_caches()
+
+
+def clear_caches():
+    """Clear caches used in opensearch service and factories"""
+    check_hybrid_search_enabled.cache_clear()
+    # the instance of check_hybrid_search_enabled used in utils.py
+    # is different and must be cleared separately
+    check_hybrid_search_enabled_utils.cache_clear()
+    opensearch_client().indices.delete(index="*")
+
+
+@responses.activate
+def test_embed_text_success(settings):
+    """Test embed_text retrieval is successful"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    text = "canine pet"
+
+    embedding = embed_text(text)
+
+    assert embedding == albert_embedding_response.response["data"][0]["embedding"]
+
+
+@responses.activate
+def test_embed_401_http_error(settings, caplog):
+    """Test embed_text does not crash and returns None on 401 error"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        status=401,
+        body=json_dumps({"message": "Authentication failed."}),
+    )
+    text = "canine pet"
+
+    with caplog.at_level(logging.WARNING):
+        embedding = embed_text(text)
+
+    assert any(
+        "embedding API request failed: 401 Client Error: Unauthorized" in message
+        for message in caplog.messages
+    )
+
+    assert embedding is None
+
+
+@responses.activate
+def test_embed_500_http_error(settings, caplog):
+    """Test embed_text does not crash and returns None on 500 error"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        status=500,
+        body=json_dumps({"message": "Internal server error."}),
+    )
+    text = "canine pet"
+
+    with caplog.at_level(logging.WARNING):
+        embedding = embed_text(text)
+
+    assert any(
+        "embedding API request failed: 500 Server Error: Internal Server Error"
+        in message
+        for message in caplog.messages
+    )
+
+    assert embedding is None
+
+
+@responses.activate
+def test_embed_wrong_format(settings, caplog):
+    """Test embed_text does not crash and returns None if api returns a wrong format"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json={"wrong": "format"},
+        status=200,
+    )
+    text = "canine pet"
+
+    with caplog.at_level(logging.WARNING):
+        embedding = embed_text(text)
+
+    assert any(
+        "unexpected embedding response format" in message for message in caplog.messages
+    )
+
+    assert embedding is None

src/backend/core/tests/test_indexing.py

@@ -0,0 +1,227 @@
+"""
+Test suite for opensearch indexing service
+"""
+
+import pytest
+import responses
+
+from core.services.indexing import detect_language_code, ensure_index_exists
+from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
+
+from .mock import albert_embedding_response
+from .utils import (
+    check_hybrid_search_enabled as check_hybrid_search_enabled_utils,
+)
+from .utils import (
+    enable_hybrid_search,
+)
+
+pytestmark = pytest.mark.django_db
+
+
+SERVICE_NAME = "test-service"
+
+
+@pytest.fixture(autouse=True)
+def before_each():
+    """Clear caches before each test"""
+    clear_caches()
+    yield
+    clear_caches()
+
+
+def clear_caches():
+    """Clear caches used in opensearch service and factories"""
+    check_hybrid_search_enabled.cache_clear()
+    # the instance of check_hybrid_search_enabled used in utils.py
+    # is different and must be cleared separately
+    check_hybrid_search_enabled_utils.cache_clear()
+    opensearch_client().indices.delete(index="*")
+
+
+@pytest.mark.parametrize(
+    "text, analyzer_name, expected_language_analyzer_tokens, expected_trigram_analyzer_tokens",
+    [
+        (
+            "l'éléphant a couru avec les Gens",
+            "french_analyzer",
+            # lowercase is applied ("Gens" -> "gens")
+            # asciifolding is applied ("éléphant" -> "elephant")
+            # stop words are removed ('avec', 'les')
+            # elisions are removed ("l'")
+            # stemming is applied ("gens" -> "gen")
+            ["elephant", "a", "couru", "gen"],
+            # lowercase is applied ("Gens" -> "gens")
+            # asciifolding is applied ("éléphant" -> "elephant")
+            # words smaller than 3 characters are removed ("a")
+            # trigrams are generated
+            [
+                "l'e",
+                "'el",
+                "ele",
+                "lep",
+                "eph",
+                "pha",
+                "han",
+                "ant",
+                "cou",
+                "our",
+                "uru",
+                "ave",
+                "vec",
+                "les",
+                "gen",
+                "ens",
+            ],
+        ),
+        (
+            "The Elephant is running into a café",
+            "english_analyzer",
+            # lowercase is applied ("Elephant" -> "elephant")
+            # asciifolding is applied ("café" -> "cafe")
+            # stop words are removed ("The", "into", "a")
+            # stemming is applied ("running" -> "run", "elephant" -> "eleph")
+            ["eleph", "run", "cafe"],
+            # lowercase is applied ("Gens" -> "gens")
+            # asciifolding is applied ("café" -> "cafe")
+            # trigrams are generated
+            # words smaller than 3 characters are removed ("a")
+            [
+                "the",
+                "ele",
+                "lep",
+                "eph",
+                "pha",
+                "han",
+                "ant",
+                "run",
+                "unn",
+                "nni",
+                "nin",
+                "ing",
+                "int",
+                "nto",
+                "caf",
+                "afe",
+            ],
+        ),
+        (
+            "Der Käfer läuft über die Straße",
+            "german_analyzer",
+            # lowercase is applied ("Der" -> "der", "Käfer" -> "käfer", "Straße" -> "straße")
+            # asciifolding is applied ("käfer" -> "kafer", "straße" -> "strass")
+            # stop words are removed ("Der", "die")
+            # stemming is applied ("kafer" -> "kaf")
+            ["kaf", "lauft", "uber", "strass"],
+            # lowercase is applied
+            # asciifolding is applied ("käfer" -> "kafer", "straße" -> "strasse")
+            # trigrams are generated
+            [
+                "der",
+                "kaf",
+                "afe",
+                "fer",
+                "lau",
+                "auf",
+                "uft",
+                "ube",
+                "ber",
+                "die",
+                "str",
+                "tra",
+                "ras",
+                "ass",
+                "sse",
+            ],
+        ),
+        (
+            "De Kinderen lopen naar de bakkerij",
+            "dutch_analyzer",
+            # lowercase is applied ("De" -> "de", "Kinderen" -> "kinderen")
+            # stop words are removed ("De", "naar", "de")
+            # stemming is applied ("kinderen" -> "kinder", "lopen" -> "lop")
+            ["kinder", "lop", "bakkerij"],
+            # lowercase is applied
+            # words smaller than 3 characters are removed ("de")
+            # trigrams are generated
+            [
+                "kin",
+                "ind",
+                "nde",
+                "der",
+                "ere",
+                "ren",
+                "lop",
+                "ope",
+                "pen",
+                "naa",
+                "aar",
+                "bak",
+                "akk",
+                "kke",
+                "ker",
+                "eri",
+                "rij",
+            ],
+        ),
+    ],
+)
+@responses.activate
+def test_opensearch_analyzers(
+    settings,
+    text,
+    analyzer_name,
+    expected_language_analyzer_tokens,
+    expected_trigram_analyzer_tokens,
+):
+    """Test the analyzers are correctly configured in OpenSearch"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    ensure_index_exists(SERVICE_NAME)
+
+    language_analyzer_response = opensearch_client().indices.analyze(
+        index=SERVICE_NAME,
+        body={
+            "analyzer": analyzer_name,
+            "text": text,
+        },
+    )
+    language_analyzer_tokens = [
+        token_info["token"] for token_info in language_analyzer_response["tokens"]
+    ]
+    response_trigram_analyzer = opensearch_client().indices.analyze(
+        index=SERVICE_NAME,
+        body={
+            "analyzer": "trigram_analyzer",
+            "text": text,
+        },
+    )
+    trigram_analyzer_tokens = [
+        token_info["token"] for token_info in response_trigram_analyzer["tokens"]
+    ]
+
+    assert expected_language_analyzer_tokens == language_analyzer_tokens
+    assert expected_trigram_analyzer_tokens == trigram_analyzer_tokens
+
+
+@pytest.mark.parametrize(
+    "text, expected_language_code",
+    [
+        ("This is a test sentence.", "en"),
+        ("Ceci est une phrase de test.", "fr"),
+        ("Dies ist ein Testsatz.", "de"),
+        ("Dit is een testzin.", "nl"),
+        ("Esta es una oración de prueba.", "und"),  # Spanish, unsupported
+        ("", "und"),
+        ("zefk,l", "und"),
+    ],
+)
+def test_detect_language_code(text, expected_language_code):
+    """Test detect_language_code function"""
+
+    assert detect_language_code(text) == expected_language_code

src/backend/core/tests/test_opensearch.py

@@ -1,447 +0,0 @@
-"""
-Test suite for opensearch service
-"""
-
-import logging
-import operator
-from json import dumps as json_dumps
-
-import pytest
-import responses
-
-from core import factories
-from core.services import opensearch
-from core.services.opensearch import (
-    check_hybrid_search_enabled,
-    embed_text,
-    search,
-)
-
-from .mock import albert_embedding_response
-from .utils import (
-    bulk_create_documents,
-    delete_search_pipeline,
-    enable_hybrid_search,
-    prepare_index,
-)
-from .utils import (
-    check_hybrid_search_enabled as check_hybrid_search_enabled_utils,
-)
-
-pytestmark = pytest.mark.django_db
-
-
-SERVICE_NAME = "test-service"
-
-
-def search_params(service):
-    """Build opensearch.search() parameters for tests using the service index name"""
-    return {
-        "nb_results": 20,
-        "order_by": "relevance",
-        "order_direction": "desc",
-        "search_indices": {service.index_name},
-        "reach": None,
-        "user_sub": "user_sub",
-        "groups": [],
-        "visited": [],
-    }
-
-
-@pytest.fixture(autouse=True)
-def before_each():
-    """Clear caches and delete search pipeline before each test"""
-    clear_caches()
-    yield
-    clear_caches()
-
-
-def clear_caches():
-    """Clear caches used in opensearch service and factories"""
-    check_hybrid_search_enabled.cache_clear()
-    # the instance of check_hybrid_search_enabled used in utils.py
-    # is different and must be cleared separately
-    check_hybrid_search_enabled_utils.cache_clear()
-    delete_search_pipeline()
-
-
-@responses.activate
-def test_hybrid_search_success(settings, caplog):
-    """Test the hybrid search is successful"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "canine pet"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        f"Performing hybrid search with embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    assert result["hits"]["max_score"] > 0.0
-    # hybrid search always returns a response of fixed sized sorted and scored by relevance
-    assert {hit["_source"]["title"] for hit in result["hits"]["hits"]} == {
-        doc["title"] for doc in documents
-    }
-
-
-@responses.activate
-def test_hybrid_search_without_embedded_index(settings, caplog):
-    """Test the hybrid search is successful"""
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    # index is prepared but hybrid search is not yet enable.
-    # they then won't be embedded.
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    # check embedding is None
-    indexed_documents = opensearch.opensearch_client().search(
-        index=service.index_name, body={"query": {"match_all": {}}}
-    )
-    assert indexed_documents["hits"]["hits"][0]["_source"]["embedding"] is None
-
-    # hybrid search is enabled before to do the first requests
-    enable_hybrid_search(settings)
-
-    q = "canine pet"
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    # the hybrid search is done successfully
-    assert any(
-        f"Performing hybrid search with embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    # but no match can obviously be found
-    assert result["hits"]["max_score"] == 0.0
-    assert len(result["hits"]["hits"]) == 0
-
-    # The full-text search is still functional
-    q = "wolf"
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        f"Performing hybrid search with embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert result["hits"]["hits"][0]["_source"]["title"] == q
-
-
-def test_fall_back_on_full_text_search_if_hybrid_search_disabled(settings, caplog):
-    """Test the full-text search is done when HYBRID_SEARCH_ENABLED=False"""
-    enable_hybrid_search(settings)
-    settings.HYBRID_SEARCH_ENABLED = False
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "wolf"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        "Hybrid search is disabled via HYBRID_SEARCH_ENABLED setting" in message
-        for message in caplog.messages
-    )
-    assert any(
-        f"Performing full-text search without embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert result["hits"]["hits"][0]["_source"]["title"] == "wolf"
-
-
-@responses.activate
-def test_fall_back_on_full_text_search_if_embedding_api_fails(settings, caplog):
-    """Test the full-text search is done when the embedding api fails"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        status=401,
-        body=json_dumps({"message": "Authentication failed."}),
-    )
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "wolf"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        "embedding API request failed: 401 Client Error: Unauthorized" in message
-        for message in caplog.messages
-    )
-    assert any(
-        f"Performing full-text search without embedding: {q}" in message
-        for message in caplog.messages
-    )
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert result["hits"]["hits"][0]["_source"]["title"] == "wolf"
-
-
-@responses.activate
-def test_fall_back_on_full_text_search_if_variable_are_missing(settings, caplog):
-    """Test the full-text search is done when variables are missing for hybrid search"""
-    enable_hybrid_search(settings)
-    del settings.HYBRID_SEARCH_WEIGHTS
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "wolf"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        "Missing variables for hybrid search: HYBRID_SEARCH_WEIGHTS" in message
-        for message in caplog.messages
-    )
-    assert any(
-        f"Performing full-text search without embedding: {q}" in message
-        for message in caplog.messages
-    )
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert result["hits"]["hits"][0]["_source"]["title"] == "wolf"
-
-
-@responses.activate
-def test_match_all(settings, caplog):
-    """Test match all when q='*' and no semantic search is needed"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "*"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any("Performing match_all query" in message for message in caplog.messages)
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 3
-
-
-@responses.activate
-def test_search_ordering_by_relevance(settings, caplog):
-    """Test the hybrid supports ordering by relevance asc and desc"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    q = "canine pet"
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    for direction in ["asc", "desc"]:
-        with caplog.at_level(logging.INFO):
-            result = search(
-                q=q, **{**search_params(service), "order_direction": direction}
-            )
-
-        # Check that results are sorted by score as expected
-        hits = result["hits"]["hits"]
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(hits) - 1):
-            assert compare(hits[i]["_score"], hits[i + 1]["_score"])
-
-
-@responses.activate
-def test_hybrid_search_number_of_matches(settings):
-    """
-    In this test full-text search always return 0 documents.
-    The test checks the number of hits returned by hybrid search with different k values.
-    """
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "pony"  # full-text matches 0 document
-    for nb_results in [1, 2, 3]:  # semantic should match k documents
-        result = search(q=q, **{**search_params(service), "nb_results": nb_results})
-        assert len(result["hits"]["hits"]) == nb_results
-
-
-@responses.activate
-def test_embed_text_success(settings):
-    """Test embed_text retrieval is successful"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    text = "canine pet"
-
-    embedding = embed_text(text)
-
-    assert embedding == albert_embedding_response.response["data"][0]["embedding"]
-
-
-@responses.activate
-def test_embed_401_http_error(settings, caplog):
-    """Test embed_text does not crash and returns None on 401 error"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        status=401,
-        body=json_dumps({"message": "Authentication failed."}),
-    )
-    text = "canine pet"
-
-    with caplog.at_level(logging.WARNING):
-        embedding = embed_text(text)
-
-    assert any(
-        "embedding API request failed: 401 Client Error: Unauthorized" in message
-        for message in caplog.messages
-    )
-
-    assert embedding is None
-
-
-@responses.activate
-def test_embed_500_http_error(settings, caplog):
-    """Test embed_text does not crash and returns None on 500 error"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        status=500,
-        body=json_dumps({"message": "Internal server error."}),
-    )
-    text = "canine pet"
-
-    with caplog.at_level(logging.WARNING):
-        embedding = embed_text(text)
-
-    assert any(
-        "embedding API request failed: 500 Server Error: Internal Server Error"
-        in message
-        for message in caplog.messages
-    )
-
-    assert embedding is None
-
-
-@responses.activate
-def test_embed_wrong_format(settings, caplog):
-    """Test embed_text does not crash and returns None if api returns a wrong format"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json={"wrong": "format"},
-        status=200,
-    )
-    text = "canine pet"
-
-    with caplog.at_level(logging.WARNING):
-        embedding = embed_text(text)
-
-    assert any(
-        "unexpected embedding response format" in message for message in caplog.messages
-    )
-
-    assert embedding is None

src/backend/core/tests/test_search.py

@@ -0,0 +1,717 @@
+"""
+Test suite for opensearch search service
+"""
+
+import datetime
+import logging
+from json import dumps as json_dumps
+
+import pytest
+import responses
+
+from core import enums, factories
+from core.services import opensearch
+from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
+from core.services.search import search
+from core.utils import bulk_create_documents, delete_search_pipeline, prepare_index
+
+from .mock import albert_embedding_response
+from .utils import (
+    check_hybrid_search_enabled as check_hybrid_search_enabled_utils,
+)
+from .utils import (
+    enable_hybrid_search,
+)
+
+pytestmark = pytest.mark.django_db
+
+SERVICE_NAME = "test-service"
+
+
+def search_params(service):
+    """Build opensearch.search() parameters for tests using the service index name"""
+    return {
+        "nb_results": 20,
+        "search_indices": {service.index_name},
+        "reach": None,
+        "user_sub": "user_sub",
+        "groups": [],
+        "visited": [],
+        "tags": [],
+        "search_type": enums.SearchTypeEnum.HYBRID,
+    }
+
+
+@pytest.fixture(autouse=True)
+def before_each():
+    """Clear caches and delete search pipeline before each test"""
+    clear_caches()
+    yield
+    clear_caches()
+
+
+def clear_caches():
+    """Clear caches used in opensearch service and factories"""
+    check_hybrid_search_enabled.cache_clear()
+    # the instance of check_hybrid_search_enabled used in utils.py
+    # is different and must be cleared separately
+    check_hybrid_search_enabled_utils.cache_clear()
+    delete_search_pipeline()
+    opensearch_client().indices.delete(index="*")
+
+
+@responses.activate
+def test_hybrid_search_success(settings, caplog):
+    """Test the hybrid search is successful"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    q = "canine pet"
+    with caplog.at_level(logging.INFO):
+        result = search(q=q, **search_params(service))
+
+    assert any(
+        f"Performing hybrid search with embedding: {q}" in message
+        for message in caplog.messages
+    )
+
+    assert result["hits"]["max_score"] > 0.0
+    # hybrid search always returns a response of fixed sized sorted and scored by relevance
+    assert {hit["_source"]["title.en"] for hit in result["hits"]["hits"]} == {
+        doc["title"] for doc in documents
+    }
+
+
+@responses.activate
+def test_hybrid_search_without_embedded_index(settings, caplog):
+    """Test the hybrid search is successful"""
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves"},
+            {"title": "dog", "content": "dogs"},
+            {"title": "cat", "content": "cats"},
+        ]
+    )
+    # index is prepared but hybrid search is not yet enable.
+    # they then won't be embedded.
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    # check embedding is None
+    indexed_documents = opensearch.opensearch_client().search(
+        index=service.index_name, body={"query": {"match_all": {}}}
+    )
+    assert indexed_documents["hits"]["hits"][0]["_source"]["chunks"] is None
+
+    # hybrid search is enabled before to do the first requests
+    enable_hybrid_search(settings)
+
+    q = "canine pet"
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    with caplog.at_level(logging.INFO):
+        result = search(q=q, **search_params(service))
+
+    # the hybrid search is done successfully
+    assert any(
+        f"Performing hybrid search with embedding: {q}" in message
+        for message in caplog.messages
+    )
+
+    # but no match can obviously be found
+    assert result["hits"]["max_score"] == 0.0
+    assert len(result["hits"]["hits"]) == 0
+
+    # The full-text search is still functional
+    q = "wolf"
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    with caplog.at_level(logging.INFO):
+        result = search(q=q, **search_params(service))
+
+    assert any(
+        f"Performing hybrid search with embedding: {q}" in message
+        for message in caplog.messages
+    )
+
+    assert result["hits"]["max_score"] > 0.0
+    assert len(result["hits"]["hits"]) == 1
+    assert (
+        result["hits"]["hits"][0]["_source"][
+            f"title.{settings.UNDETERMINED_LANGUAGE_CODE}"
+        ]
+        == q
+    )
+
+
+def test_fall_back_on_full_text_search_if_hybrid_search_disabled(settings, caplog):
+    """Test the full-text search is done when HYBRID_SEARCH_ENABLED=False"""
+    enable_hybrid_search(settings)
+    settings.HYBRID_SEARCH_ENABLED = False
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    q = "wolf"
+    with caplog.at_level(logging.INFO):
+        result = search(q=q, **search_params(service))
+
+    assert any(
+        "Hybrid search is disabled via HYBRID_SEARCH_ENABLED setting" in message
+        for message in caplog.messages
+    )
+    assert any(
+        f"Performing full-text search without embedding: {q}" in message
+        for message in caplog.messages
+    )
+
+    assert result["hits"]["max_score"] > 0.0
+    assert len(result["hits"]["hits"]) == 1
+    assert result["hits"]["hits"][0]["_source"]["title.en"] == "wolf"
+
+
+@responses.activate
+def test_force_full_text_search_with_search_type_parameter(settings, caplog):
+    """Test the full-text search is done when search_type=FULL_TEXT even if hybrid is enabled"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    q = "wolf"
+    with caplog.at_level(logging.INFO):
+        result = search(
+            q=q,
+            **{**search_params(service), "search_type": enums.SearchTypeEnum.FULL_TEXT},
+        )
+
+    assert any(
+        f"Performing full-text search without embedding: {q}" in message
+        for message in caplog.messages
+    )
+
+    assert result["hits"]["max_score"] > 0.0
+    assert len(result["hits"]["hits"]) == 1
+    assert result["hits"]["hits"][0]["_source"]["title.en"] == "wolf"
+
+
+def test_request_hybrid_search_when_server_has_it_disabled(settings, caplog):
+    """Test warning when hybrid search is requested but disabled on server"""
+    enable_hybrid_search(settings)
+    settings.HYBRID_SEARCH_ENABLED = False
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    q = "canine pet"
+    with caplog.at_level(logging.INFO):
+        search(
+            q=q,
+            **{**search_params(service), "search_type": enums.SearchTypeEnum.HYBRID},
+        )
+
+    assert any(
+        "Hybrid search was requested (search_type=hybrid) but is disabled on server"
+        in message
+        for message in caplog.messages
+    )
+    assert any(
+        f"Performing full-text search without embedding: {q}" in message
+        for message in caplog.messages
+    )
+
+
+@responses.activate
+def test_api_documents_search_with_search_type_hybrid(settings, caplog):
+    """Test API with search_type=hybrid uses hybrid search when enabled"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    service = factories.ServiceFactory()
+
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+        ]
+    )
+    prepare_index(service.index_name, documents)
+
+    q = "canine pet"
+    with caplog.at_level(logging.INFO):
+        result = search(
+            q=q,
+            **{**search_params(service), "search_type": enums.SearchTypeEnum.HYBRID},
+        )
+
+    assert any(
+        f"Performing hybrid search with embedding: {q}" in message
+        for message in caplog.messages
+    )
+
+    assert result["hits"]["max_score"] > 0.0
+    # hybrid search always returns a response of fixed sized sorted and scored by relevance
+    assert {hit["_source"]["title.en"] for hit in result["hits"]["hits"]} == {
+        doc["title"] for doc in documents
+    }
+
+
+@responses.activate
+def test_fall_back_on_full_text_search_if_embedding_api_fails(settings, caplog):
+    """Test the full-text search is done when the embedding api fails"""
+
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        status=401,
+        body=json_dumps({"message": "Authentication failed."}),
+    )
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    q = "wolf"
+    with caplog.at_level(logging.INFO):
+        result = search(q=q, **search_params(service))
+
+    assert any(
+        "embedding API request failed: 401 Client Error: Unauthorized" in message
+        for message in caplog.messages
+    )
+    assert any(
+        f"Performing full-text search without embedding: {q}" in message
+        for message in caplog.messages
+    )
+    assert result["hits"]["max_score"] > 0.0
+    assert len(result["hits"]["hits"]) == 1
+    assert result["hits"]["hits"][0]["_source"]["title.en"] == "wolf"
+
+
+@responses.activate
+def test_fall_back_on_full_text_search_if_variable_are_missing(settings, caplog):
+    """Test the full-text search is done when variables are missing for hybrid search"""
+    enable_hybrid_search(settings)
+    del settings.HYBRID_SEARCH_WEIGHTS
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    q = "wolf"
+    with caplog.at_level(logging.INFO):
+        result = search(q=q, **search_params(service))
+
+    assert any(
+        "Missing variables for hybrid search: HYBRID_SEARCH_WEIGHTS" in message
+        for message in caplog.messages
+    )
+    assert any(
+        f"Performing full-text search without embedding: {q}" in message
+        for message in caplog.messages
+    )
+    assert result["hits"]["max_score"] > 0.0
+    assert len(result["hits"]["hits"]) == 1
+    assert result["hits"]["hits"][0]["_source"]["title.en"] == "wolf"
+
+
+@responses.activate
+def test_match_all(settings, caplog):
+    """Test match all when q='*' and no semantic search is needed"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    q = "*"
+    with caplog.at_level(logging.INFO):
+        result = search(q=q, **search_params(service))
+
+    assert any("Performing match_all query" in message for message in caplog.messages)
+    assert result["hits"]["max_score"] > 0.0
+    assert len(result["hits"]["hits"]) == 3
+
+
+@responses.activate
+def test_hybrid_search_number_of_matches(settings):
+    """
+    In this test full-text search always return 0 documents.
+    The test checks the number of hits returned by hybrid search with different k values.
+    """
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    q = "pony"  # full-text matches 0 document
+    for nb_results in [1, 2, 3]:  # semantic should match k documents
+        result = search(q=q, **{**search_params(service), "nb_results": nb_results})
+        assert len(result["hits"]["hits"]) == nb_results
+
+
+def test_search_filtering_by_single_tag():
+    """Test filtering documents by a single tag"""
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    documents_to_search = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["tag-to-search"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["tag-to-search", "tag-to-filter"]
+        ),
+    ]
+    documents_to_filter = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["tag-to-filter"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+        ),
+    ]
+    expected_ids = {str(doc["id"]) for doc in documents_to_search}
+
+    prepare_index(service.index_name, documents_to_search + documents_to_filter)
+
+    # Search for documents with tag-to-search tag
+    result = search(q="*", **{**search_params(service), "tags": ["tag-to-search"]})
+    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
+
+    assert result["hits"]["total"]["value"] == len(documents_to_search)
+    assert returned_ids == expected_ids
+
+
+def test_search_filtering_by_multiple_tags():
+    """Test filtering documents by multiple tags (OR logic)"""
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    documents_to_search = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["tag-to-search-1"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["tag-to-search-1", "tag-to-filter"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["tag-to-search-2", "tag-to-filter"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["tag-to-search-1", "tag-to-search-2"]
+        ),
+    ]
+    documents_to_filter = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], tags=["tag-to-filter"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+        ),
+    ]
+    expected_ids = {str(doc["id"]) for doc in documents_to_search}
+
+    prepare_index(service.index_name, documents_to_search + documents_to_filter)
+
+    # Search for documents with tag-to-search-1 OR tag-to-search-2 tags
+    result = search(
+        q="*",
+        **{**search_params(service), "tags": ["tag-to-search-1", "tag-to-search-2"]},
+    )
+    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
+
+    assert result["hits"]["total"]["value"] == len(documents_to_search)
+    assert returned_ids == expected_ids
+
+
+def test_search_no_tags_filter_returns_all():
+    """Test that not providing tags filter returns all documents"""
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    documents = bulk_create_documents(
+        [
+            {
+                "title": "Document without tags",
+                "content": "Untagged document",
+            },
+            {
+                "title": "Document with tags",
+                "content": "Tagged document",
+                "tags": ["tag-to-search"],
+            },
+        ]
+    )
+    prepare_index(service.index_name, documents)
+
+    # Search without tags filter
+    result = search(q="*", **search_params(service))
+
+    assert result["hits"]["total"]["value"] == len(documents)
+
+
+def test_search_filtering_by_tag_and_query():
+    """Test filtering documents by both tag and query text"""
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    documents_to_search = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], title="title to search", tags=["tag-to-search"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to search 1",
+            tags=["tag-to-search", "tag-to-filter"],
+        ),
+    ]
+    documents_to_filter = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], title="title to filter", tags=["tag-to-filter"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], title="title to filter 1", tags=["tag-to-filter"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to search 2",
+        ),
+    ]
+    expected_ids = {str(doc["id"]) for doc in documents_to_search}
+
+    prepare_index(service.index_name, documents_to_search + documents_to_filter)
+
+    # Search with both query and tag filter
+    result = search(q="search", **{**search_params(service), "tags": ["tag-to-search"]})
+    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
+
+    assert result["hits"]["total"]["value"] == len(documents_to_search)
+    assert returned_ids == expected_ids
+
+
+def test_search_filtering_by_path():
+    """Test filtering documents by path prefix"""
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    documents_to_search = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], path="path/to/search/doc-1"
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], path="path/to/search/doc-2"
+        ),
+    ]
+    documents_to_filter = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], path="path/to/filter/doc-3"
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+        ),
+    ]
+    expected_ids = {str(doc["id"]) for doc in documents_to_search}
+
+    prepare_index(service.index_name, documents_to_search + documents_to_filter)
+
+    path_filter = "path/to/search"
+    result = search(q="*", **{**search_params(service), "path": path_filter})
+    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
+
+    assert result["hits"]["total"]["value"] == len(documents_to_search)
+    assert returned_ids == expected_ids
+
+
+def test_search_filtering_by_query_path_and_tag():
+    """Test filtering documents by query text, path prefix and tag combined"""
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    documents_to_search = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to search 0",
+            path="path/to/search-0",
+            tags=["tag-to-search"],
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to search 1",
+            path="path/to/search/doc1",
+            tags=["tag-to-search", "tag-to-filter"],
+        ),
+    ]
+    documents_to_filter = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to filter",
+            path="path/to/search/doc-3",
+            tags=["tag-to-search"],
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to search 4",
+            path="path/to/filter/doc-4",
+            tags=["tag-to-search"],
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to search 4",
+            path="path/to/search/doc-4",
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"], title="title to search 5", tags=["tag-to-search"]
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to search 6",
+            path="path/to/search/doc-6",
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="title to search 7",
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="",
+            path="path/to/search/doc-8",
+            tags=["tag-to-search"],
+        ),
+    ]
+    expected_ids = {str(doc["id"]) for doc in documents_to_search}
+
+    prepare_index(service.index_name, documents_to_search + documents_to_filter)
+
+    # Search with query, path and tag filters combined
+    result = search(
+        q="search",
+        **{
+            **search_params(service),
+            "path": "path/to/search",
+            "tags": ["tag-to-search"],
+        },
+    )
+    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
+
+    assert result["hits"]["total"]["value"] == len(documents_to_search)
+    assert returned_ids == expected_ids
+
+
+def test_search_with_rescore(settings):
+    """Test rescore feature"""
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    today = datetime.datetime.today()
+    forty_days_ago = today - datetime.timedelta(days=40)
+    documents = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="my first document",
+            created_at=today,
+            updated_at=today,
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="another document",
+            created_at=forty_days_ago,
+            updated_at=forty_days_ago,
+        ),
+    ]
+    prepare_index(service.index_name, documents)
+
+    # set a cray big RESCORE_UPDATED_AT_WEIGHT to demonstrate the effect of boosting on rescores
+    settings.RESCORE_UPDATED_AT_WEIGHT = 200
+
+    results = search(
+        q="another document",
+        **{
+            **search_params(service),
+            "rescore": True,
+        },
+    )
+
+    hits = results["hits"]["hits"]
+    # the first document is ranked first because it more recent
+    # even though the second one matches the query better
+    assert hits[0]["_source"]["title.en"] == "my first document"
+    assert hits[1]["_source"]["title.en"] == "another document"

src/backend/core/tests/test_selftests.py

@@ -0,0 +1,145 @@
+"""Tests for the selftest system."""
+
+from core.selftests import SelfTest, SelfTestRegistry, SelfTestResult
+
+
+class DummySuccessTest(SelfTest):
+    """A test that always succeeds."""
+
+    name = "Dummy Success Test"
+    description = "A test that always passes"
+
+    def run(self) -> SelfTestResult:
+        return SelfTestResult(
+            name=self.name,
+            success=True,
+            message="Test passed successfully",
+            duration_ms=10.0,
+        )
+
+
+class DummyFailureTest(SelfTest):
+    """A test that always fails."""
+
+    name = "Dummy Failure Test"
+    description = "A test that always fails"
+
+    def run(self) -> SelfTestResult:
+        return SelfTestResult(
+            name=self.name,
+            success=False,
+            message="Test failed as expected",
+            duration_ms=5.0,
+        )
+
+
+class DummyExceptionTest(SelfTest):
+    """A test that raises an exception."""
+
+    name = "Dummy Exception Test"
+    description = "A test that throws an exception"
+
+    def run(self) -> SelfTestResult:
+        raise RuntimeError("This test is designed to fail")
+
+
+def test_register_test():
+    """Test that a test can be registered."""
+    registry = SelfTestRegistry()
+    registry.register(DummySuccessTest)
+
+    tests = registry.get_all_tests()
+
+    assert len(tests) == 1
+    assert isinstance(tests[0], DummySuccessTest)
+
+
+def test_register_multiple_tests():
+    """Test that multiple tests can be registered."""
+    registry = SelfTestRegistry()
+    registry.register(DummySuccessTest)
+    registry.register(DummyFailureTest)
+
+    tests = registry.get_all_tests()
+
+    assert len(tests) == 2
+
+
+def test_unregister_test():
+    """Test that a test can be unregistered."""
+    registry = SelfTestRegistry()
+    registry.register(DummySuccessTest)
+    registry.unregister(DummySuccessTest)
+
+    tests = registry.get_all_tests()
+
+    assert len(tests) == 0
+
+
+def test_run_all_success():
+    """Test running all tests when all pass."""
+    registry = SelfTestRegistry()
+    registry.register(DummySuccessTest)
+
+    results = registry.run_all()
+
+    assert len(results) == 1
+    assert results[0].success is True
+
+
+def test_run_all_mixed():
+    """Test running all tests with mixed results."""
+    registry = SelfTestRegistry()
+    registry.register(DummySuccessTest)
+    registry.register(DummyFailureTest)
+
+    results = registry.run_all()
+
+    assert len(results) == 2
+    assert results[0].success is True
+    assert results[1].success is False
+
+
+def test_run_all_with_exception():
+    """Test that exceptions are caught and converted to failures."""
+    registry = SelfTestRegistry()
+    registry.register(DummyExceptionTest)
+
+    results = registry.run_all()
+
+    assert len(results) == 1
+    assert results[0].success is False
+    assert "exception" in results[0].message.lower()
+
+
+def test_result_to_dict():
+    """Test converting a result to a dictionary."""
+    result = SelfTestResult(
+        name="Test Name",
+        success=True,
+        message="Test message",
+        details={"key": "value"},
+        duration_ms=100.5,
+    )
+
+    result_dict = result.to_dict()
+
+    assert result_dict["name"] == "Test Name"
+    assert result_dict["success"] is True
+    assert result_dict["message"] == "Test message"
+    assert result_dict["details"]["key"] == "value"
+    assert result_dict["duration_ms"] == 100.5
+
+
+def test_result_without_optional_fields():
+    """Test creating a result without optional fields."""
+    result = SelfTestResult(
+        name="Test Name",
+        success=False,
+        message="Test failed",
+    )
+
+    result_dict = result.to_dict()
+
+    assert result_dict["details"] == {}
+    assert result_dict["duration_ms"] is None

src/backend/core/tests/utils.py

@@ -1,28 +1,16 @@
-"""Tests Service model for find's core app."""
+"""Utility functions for Test."""
 
 import base64
 import json
 import logging
 from functools import partial
-from typing import List
 
-from django.conf import settings as django_settings
-
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric import rsa
-from joserfc import jwe as jose_jwe
-from joserfc import jwt as jose_jwt
-from joserfc.jwk import RSAKey
-from jwt.utils import to_base64url_uint
-from opensearchpy.exceptions import NotFoundError
-from opensearchpy.helpers import bulk
-
-from core import factories
 from core.management.commands.create_search_pipeline import (
     ensure_search_pipeline_exists,
 )
-from core.services import opensearch
-from core.services.opensearch import check_hybrid_search_enabled
+from core.services.opensearch import (
+    check_hybrid_search_enabled,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -42,58 +30,6 @@ def enable_hybrid_search(settings):
     ensure_search_pipeline_exists()
 
 
-def bulk_create_documents(document_payloads):
-    """Create documents in bulk from payloads"""
-    return [
-        factories.DocumentSchemaFactory.build(**document_payload, users=["user_sub"])
-        for document_payload in document_payloads
-    ]
-
-
-def delete_search_pipeline():
-    """Delete the hybrid search pipeline if it exists"""
-    try:
-        opensearch.opensearch_client().transport.perform_request(
-            method="DELETE",
-            url=f"/_search/pipeline/{django_settings.HYBRID_SEARCH_PIPELINE_ID}",
-        )
-    except NotFoundError:
-        logger.info("Search pipeline not found, nothing to delete.")
-
-
-def prepare_index(index_name, documents: List):
-    """Prepare the search index before testing a query on it."""
-    opensearch.ensure_index_exists(index_name)
-
-    # Index new documents
-    actions = [
-        {
-            "_op_type": "index",
-            "_index": index_name,
-            "_id": document["id"],
-            "_source": {
-                **{k: v for k, v in document.items() if k != "id"},
-                "embedding": opensearch.embed_text(
-                    opensearch.format_document(document["title"], document["content"])
-                )
-                if check_hybrid_search_enabled()
-                else None,
-                "embedding_model": django_settings.EMBEDDING_API_MODEL_NAME
-                if check_hybrid_search_enabled()
-                else None,
-            },
-        }
-        for document in documents
-    ]
-    bulk(opensearch.opensearch_client(), actions)
-
-    # Force refresh again so all changes are visible to search
-    opensearch.opensearch_client().indices.refresh(index=index_name)
-
-    count = opensearch.opensearch_client().count(index=index_name)["count"]
-    assert count == len(documents), f"Expected {len(documents)}, got {count}"
-
-
 def build_authorization_bearer(token="some_token"):
     """
     Build an Authorization Bearer header value from a token.
@@ -107,116 +43,6 @@ def build_authorization_bearer(token="some_token"):
     return base64.b64encode(token.encode("utf-8")).decode("utf-8")
 
 
-def setup_oicd_jwt_resource_server(
-    responses,
-    settings,
-    sub="some_sub",
-    audience="some_client_id",
-):
-    """
-    Setup settings for a resource server with JWT backend.
-    Simulate an encrypted token introspection.
-    NOTE : Use it with @responses.activate or the fake introspection view will not work.
-    """
-    token_data = {
-        "sub": sub,
-        "iss": "https://oidc.example.com",
-        "aud": audience,
-        "client_id": "some_service_provider",
-        "scope": "docs",
-        "active": True,
-    }
-
-    private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
-
-    unencrypted_pem_private_key = private_key.private_bytes(
-        encoding=serialization.Encoding.PEM,
-        format=serialization.PrivateFormat.TraditionalOpenSSL,
-        encryption_algorithm=serialization.NoEncryption(),
-    )
-
-    pem_public_key = private_key.public_key().public_bytes(
-        encoding=serialization.Encoding.PEM,
-        format=serialization.PublicFormat.SubjectPublicKeyInfo,
-    )
-
-    settings.OIDC_RS_PRIVATE_KEY_STR = unencrypted_pem_private_key.decode("utf-8")
-    settings.OIDC_RS_ENCRYPTION_KEY_TYPE = "RSA"
-    settings.OIDC_RS_ENCRYPTION_ENCODING = "A256GCM"
-    settings.OIDC_RS_ENCRYPTION_ALGO = "RSA-OAEP"
-    settings.OIDC_RS_SIGNING_ALGO = "RS256"
-    settings.OIDC_RS_CLIENT_ID = audience
-    settings.OIDC_RS_CLIENT_SECRET = "some_client_secret"
-    settings.OIDC_RS_SCOPES = ["openid", "docs", "email"]
-
-    settings.OIDC_OP_URL = "https://oidc.example.com"
-    settings.OIDC_OP_JWKS_ENDPOINT = "https://oidc.example.com/jwks"
-    settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
-
-    settings.OIDC_VERIFY_SSL = False
-    settings.OIDC_TIMEOUT = 5
-    settings.OIDC_PROXY = None
-    settings.OIDC_CREATE_USER = False
-
-    # Mock the JWKS endpoint
-    public_numbers = private_key.public_key().public_numbers()
-    responses.add(
-        responses.GET,
-        settings.OIDC_OP_JWKS_ENDPOINT,
-        body=json.dumps(
-            {
-                "keys": [
-                    {
-                        "kty": settings.OIDC_RS_ENCRYPTION_KEY_TYPE,
-                        "alg": settings.OIDC_RS_SIGNING_ALGO,
-                        "use": "sig",
-                        "kid": "1234567890",
-                        "n": to_base64url_uint(public_numbers.n).decode("ascii"),
-                        "e": to_base64url_uint(public_numbers.e).decode("ascii"),
-                    }
-                ]
-            }
-        ),
-    )
-
-    def encrypt_jwt(json_data):
-        """Encrypt the JWT token for the backend to decrypt."""
-        token = jose_jwt.encode(
-            {
-                "kid": "1234567890",
-                "alg": settings.OIDC_RS_SIGNING_ALGO,
-            },
-            json_data,
-            RSAKey.import_key(unencrypted_pem_private_key),
-            algorithms=[settings.OIDC_RS_SIGNING_ALGO],
-        )
-
-        return jose_jwe.encrypt_compact(
-            protected={
-                "alg": settings.OIDC_RS_ENCRYPTION_ALGO,
-                "enc": settings.OIDC_RS_ENCRYPTION_ENCODING,
-            },
-            plaintext=token,
-            public_key=RSAKey.import_key(pem_public_key),
-            algorithms=[
-                settings.OIDC_RS_ENCRYPTION_ALGO,
-                settings.OIDC_RS_ENCRYPTION_ENCODING,
-            ],
-        )
-
-    responses.add(
-        responses.POST,
-        "https://oidc.example.com/introspect",
-        body=encrypt_jwt(
-            {
-                "iss": "https://oidc.example.com",
-                "aud": audience,  # settings.OIDC_RS_CLIENT_ID
-                "token_introspection": token_data,
-            }
-        ),
-    )
-
-
 def setup_oicd_resource_server(
     responses,
     settings,
@@ -257,12 +83,12 @@ def setup_oicd_resource_server(
     if callable(introspect):
         responses.add_callback(
             responses.POST,
-            "https://oidc.example.com/introspect",
+            settings.OIDC_OP_INTROSPECTION_ENDPOINT,
             callback=partial(introspect, user_info=token_data),
         )
     else:
         responses.add(
             responses.POST,
-            "https://oidc.example.com/introspect",
+            settings.OIDC_OP_INTROSPECTION_ENDPOINT,
             body=json.dumps(token_data),
         )

src/backend/core/urls.py

@@ -2,10 +2,11 @@
 
 from django.urls import include, path
 
-from .views import IndexDocumentView, SearchDocumentView
+from .views import DeleteDocumentsView, IndexDocumentView, SearchDocumentView
 
 urlpatterns = [
     path("documents/index/", IndexDocumentView.as_view(), name="document"),
     path("documents/search/", SearchDocumentView.as_view(), name="document"),
+    path("documents/delete/", DeleteDocumentsView.as_view(), name="document"),
     path("", include("lasuite.oidc_resource_server.urls")),
 ]

src/backend/core/utils.py

@@ -0,0 +1,80 @@
+"""Tests Service model for find's core app."""
+
+import logging
+from typing import List
+
+from django.conf import settings as django_settings
+
+from opensearchpy.exceptions import NotFoundError
+from opensearchpy.helpers import bulk
+
+from core import factories
+from core.services.indexing import ensure_index_exists, prepare_document_for_indexing
+from core.services.opensearch import opensearch_client
+
+logger = logging.getLogger(__name__)
+
+
+def bulk_create_documents(document_payloads):
+    """Create documents in bulk from payloads"""
+    return [
+        factories.DocumentSchemaFactory.build(**document_payload, users=["user_sub"])
+        for document_payload in document_payloads
+    ]
+
+
+def delete_search_pipeline():
+    """Delete the hybrid search pipeline if it exists"""
+    logger.info(
+        "Deleting search pipeline %s", django_settings.HYBRID_SEARCH_PIPELINE_ID
+    )
+
+    try:
+        opensearch_client().transport.perform_request(
+            method="DELETE",
+            url=f"/_search/pipeline/{django_settings.HYBRID_SEARCH_PIPELINE_ID}",
+        )
+    except NotFoundError:
+        logger.info("Search pipeline not found, nothing to delete.")
+
+
+def delete_index(index_name):
+    """Delete the hybrid search pipeline if it exists"""
+    logger.info("Deleting Index %s", index_name)
+
+    try:
+        opensearch_client().indices.delete(index=index_name)
+    except NotFoundError:
+        logger.info("Search pipeline %s not found, nothing to delete.", index_name)
+
+
+def prepare_index(index_name, documents: List):
+    """Prepare the search index before testing a query on it."""
+    logger.info("Preparing index %s with %d documents", index_name, len(documents))
+
+    ensure_index_exists(index_name)
+    actions = [
+        {
+            "_op_type": "index",
+            "_index": index_name,
+            "_id": document["id"],
+            "_source": prepare_document_for_indexing(document),
+        }
+        for document in documents
+    ]
+    bulk(opensearch_client(), actions)
+    opensearch_client().indices.refresh(index=index_name)
+
+
+def get_language_value(source, language_field):
+    """
+    extract the value of the language field with the correct language_code extension.
+    "title" and "content" have extensions like "title.en" or "title.fr".
+    get_language_value will return the value regardless of the extension.
+    """
+    for language_code in django_settings.SUPPORTED_LANGUAGE_CODES:
+        if f"{language_field}.{language_code}" in source:
+            return source[f"{language_field}.{language_code}"]
+    raise ValueError(
+        f"No '{language_field}' field with any supported language code in object"
+    )

src/backend/core/views.py

@@ -2,7 +2,6 @@
 
 import logging
 
-from django.conf import settings
 from django.core.exceptions import SuspiciousOperation
 
 from lasuite.oidc_resource_server.authentication import ResourceServerAuthentication
@@ -13,15 +12,16 @@ from rest_framework.response import Response
 
 from . import schemas
 from .authentication import ServiceTokenAuthentication
-from .models import Service, get_opensearch_index_name
+from .enums import SearchTypeEnum
 from .permissions import IsAuthAuthenticated
-from .services.opensearch import (
-    check_hybrid_search_enabled,
-    embed_document,
+from .services.indexing import (
     ensure_index_exists,
-    opensearch_client,
-    search,
+    get_opensearch_indices,
+    prepare_document_for_indexing,
 )
+from .services.opensearch import check_hybrid_search_enabled, opensearch_client
+from .services.search import search
+from .utils import get_language_value
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +37,6 @@ class IndexDocumentView(views.APIView):
     authentication_classes = [ServiceTokenAuthentication]
     permission_classes = [IsAuthAuthenticated]
 
-    # pylint: disable=too-many-locals
     def post(self, request, *args, **kwargs):
         """
         API view for indexing documents into OpenSearch index of the authenticated service.
@@ -90,76 +89,214 @@ class IndexDocumentView(views.APIView):
         opensearch_client_ = opensearch_client()
 
         if isinstance(request.data, list):
-            # Bulk indexing several documents
-            results = []
-            actions = []
-            has_errors = False
+            return self.bulk_index(request, index_name, opensearch_client_)
 
-            for i, document_data in enumerate(request.data):
-                try:
-                    document = schemas.DocumentSchema(**document_data)
-                except PydanticValidationError as excpt:
-                    errors = [
-                        {key: error[key] for key in ("msg", "type", "loc")}
-                        for error in excpt.errors()
-                    ]
-                    results.append({"index": i, "status": "error", "errors": errors})
-                    has_errors = True
-                else:
-                    document_dict = {
-                        **document.model_dump(),
-                        "embedding": embed_document(document)
-                        if check_hybrid_search_enabled()
-                        else None,
-                        "embedding_model": settings.EMBEDDING_API_MODEL_NAME
-                        if check_hybrid_search_enabled()
-                        else None,
-                    }
-                    _id = document_dict.pop("id")
-                    actions.append({"index": {"_id": _id}})
-                    actions.append(document_dict)
-                    results.append({"index": i, "_id": _id, "status": "valid"})
+        return self.single_index(request, index_name, opensearch_client_)
 
-            if has_errors:
-                return Response(results, status=status.HTTP_400_BAD_REQUEST)
+    def single_index(self, request, index_name, opensearch_client_):
+        """
+        Index a single document into OpenSearch.
 
-            # Build index if needed.
-            ensure_index_exists(index_name)
+        Args:
+            request: The HTTP request containing document data.
+            index_name: The name of the OpenSearch index.
+            opensearch_client_: The OpenSearch client instance.
 
-            response = opensearch_client_.bulk(index=index_name, body=actions)
-            for i, item in enumerate(response["items"]):
-                if item["index"]["status"] != 201:
-                    results[i]["status"] = "error"
-                    results[i]["message"] = (
-                        item["index"].get("error", {}).get("reason", "Unknown error")
-                    )
-                else:
-                    results[i]["status"] = "success"
-
-            return Response(results, status=status.HTTP_201_CREATED)
-
-        # Indexing a single document
-        document = schemas.DocumentSchema(**request.data)
-        document_dict = {
-            **document.model_dump(),
-            "embedding": embed_document(document)
-            if check_hybrid_search_enabled()
-            else None,
-            "embedding_model": settings.EMBEDDING_API_MODEL_NAME
-            if check_hybrid_search_enabled()
-            else None,
-        }
+        Returns:
+            Response: HTTP response with status and document ID.
+                - 201 Created: Returns the indexed document ID.
+                - 400 Bad Request: Returns an error message if the document is invalid.
+        """
+        document_dict = prepare_document_for_indexing(
+            schemas.DocumentSchema(**request.data).model_dump()
+        )
         _id = document_dict.pop("id")
+        logger.info(
+            "Indexing single document %s on index %s",
+            get_language_value(document_dict, "title"),
+            index_name,
+        )
 
-        # Build index if needed.
         ensure_index_exists(index_name)
-
-        opensearch_client_.index(index=index_name, body=document_dict, id=_id)
+        opensearch_client_.index(
+            index=index_name,
+            body=document_dict,
+            id=_id,
+        )
 
         return Response(
             {"status": "created", "_id": _id}, status=status.HTTP_201_CREATED
         )
 
+    # pylint: disable=too-many-locals
+    def bulk_index(self, request, index_name, opensearch_client_):
+        """
+        Index multiple documents into OpenSearch in bulk.
+
+        Args:
+            request: The HTTP request containing a list of documents.
+            index_name: The name of the OpenSearch index.
+            opensearch_client_: The OpenSearch client instance.
+
+        Returns:
+            Response: HTTP response with detailed status for each document.
+                - 201 Created: Returns status for all documents.
+                - 400 Bad Request: Returns errors if document validation fails.
+        """
+        results = []
+        actions = []
+        has_errors = False
+
+        for i, document_data in enumerate(request.data):
+            try:
+                document = schemas.DocumentSchema(**document_data)
+            except PydanticValidationError as excpt:
+                errors = [
+                    {key: error[key] for key in ("msg", "type", "loc")}
+                    for error in excpt.errors()
+                ]
+                results.append({"index": i, "status": "error", "errors": errors})
+                has_errors = True
+            else:
+                document_dict = prepare_document_for_indexing(document.model_dump())
+                logger.info(
+                    "Indexing document %s on index %s",
+                    get_language_value(document_dict, "title"),
+                    index_name,
+                )
+                _id = document_dict.pop("id")
+                actions.append({"index": {"_id": _id}})
+                actions.append(document_dict)
+                results.append({"index": i, "_id": _id, "status": "valid"})
+
+        if has_errors:
+            return Response(results, status=status.HTTP_400_BAD_REQUEST)
+
+        ensure_index_exists(index_name)
+        response = opensearch_client_.bulk(index=index_name, body=actions)
+        for i, item in enumerate(response["items"]):
+            if item["index"]["status"] != 201:
+                results[i]["status"] = "error"
+                results[i]["message"] = (
+                    item["index"].get("error", {}).get("reason", "Unknown error")
+                )
+            else:
+                results[i]["status"] = "success"
+
+        return Response(results, status=status.HTTP_201_CREATED)
+
+
+class DeleteDocumentsView(ResourceServerMixin, views.APIView):
+    """
+    API view for deleting documents from OpenSearch.
+        - Allows authenticated users to delete documents from a specified index.
+        - Users can only delete documents where they are listed in the 'users' field.
+        - Returns the count of deleted documents without revealing document existence.
+    """
+
+    authentication_classes = [ResourceServerAuthentication]
+    permission_classes = [IsAuthAuthenticated]
+
+    def post(self, request, *args, **kwargs):
+        """
+        Handle POST requests to delete documents from the specified index.
+
+        Only documents where the authenticated user is in the 'users' field will be deleted.
+
+        Body Parameters:
+        ---------------
+        service: str
+            service name to determine the index from which to delete documents.
+        document_ids : List[str], optional
+            A list of document IDs to delete from the index.
+        tags : List[str], optional
+            A list of tags to filter documents for deletion.
+
+        At least one of document_ids or tags must be provided.
+        The list of ids and the list of tags are combined with AND logic.
+
+        Returns:
+        --------
+        Response : rest_framework.response.Response
+            - 200 OK: returns a JSON object with the following keys:
+                - nb-deleted-documents: Number of documents deleted.
+                - undeleted-document-ids: sublist of param.document_ids that were not deleted.
+                Deletion may be prevented because the document does not exist,
+                because the user is not authorized to delete it or because a tag filter was used.
+            - 400 Bad Request: If parameters are invalid or missing.
+        """
+        params = schemas.DeleteDocumentsSchema(**request.data)
+        try:
+            index_name = get_opensearch_indices(
+                self._get_service_provider_audience(), services=[params.service]
+            )[0]
+        except SuspiciousOperation as e:
+            logger.error(e)
+            return Response(
+                {"detail": "Invalid request."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        logger.info(
+            "Deleting documents from index %s with filters: document_ids=%s, tags=%s",
+            index_name,
+            params.document_ids,
+            params.tags,
+        )
+
+        client = opensearch_client()
+        deletable_matches = client.search(
+            index=index_name,
+            body={
+                "query": self._build_query(
+                    self.request.user.sub,
+                    document_ids=params.document_ids,
+                    tags=params.tags,
+                )
+            },
+        )
+        deletable_ids = [hit["_id"] for hit in deletable_matches["hits"]["hits"]]
+
+        if deletable_ids:
+            response = client.delete_by_query(
+                index=index_name,
+                body={"query": {"ids": {"values": deletable_ids}}},
+            )
+            nb_deleted = response.get("deleted", 0)
+        else:
+            nb_deleted = 0
+
+        return Response(
+            {
+                "nb-deleted-documents": nb_deleted,
+                "undeleted-document-ids": [
+                    document_id
+                    for document_id in params.document_ids or []
+                    if document_id not in deletable_ids
+                ],
+            },
+            status=status.HTTP_200_OK,
+        )
+
+    def _build_query(self, user_sub, document_ids=None, tags=None):
+        """
+        Build OpenSearch query for document deletion.
+
+        Args:
+            user_sub: User subject identifier for authorization.
+            document_ids: Optional list of document IDs to filter.
+            tags: Optional list of tags to filter.
+
+        Returns:
+            Deletion OpenSearch query.
+        """
+        filters = [{"term": {"users": user_sub}}]
+        if document_ids:
+            filters.append({"ids": {"values": document_ids}})
+        if tags:
+            filters.append({"terms": {"tags": tags}})
+        return {"bool": {"must": filters}}
+
 
 class SearchDocumentView(ResourceServerMixin, views.APIView):
     """
@@ -172,27 +309,6 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
     authentication_classes = [ResourceServerAuthentication]
     permission_classes = [IsAuthAuthenticated]
 
-    @staticmethod
-    def _get_opensearch_indices(audience, services):
-        # Get request user service
-        try:
-            user_service = Service.objects.get(client_id=audience, is_active=True)
-        except Service.DoesNotExist as e:
-            logger.warning("Login failed: No service %s found", audience)
-            raise SuspiciousOperation("Service is not available") from e
-
-        # Find allowed sub-services for this service
-        allowed_services = set(user_service.services.values_list("name", flat=True))
-        allowed_services.add(user_service.name)
-
-        if services:
-            available = set(services).intersection(allowed_services)
-
-            if len(available) < len(services):
-                raise SuspiciousOperation("Some requested services are not available")
-
-        return [get_opensearch_index_name(name) for name in allowed_services]
-
     def post(self, request, *args, **kwargs):
         """
         Handle POST requests to perform a search on indexed documents with optional filtering
@@ -210,12 +326,12 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
             The search query string. This is a required parameter.
         reach : str, optional
             Filter results based on the 'reach' field.
-        order_by : str, optional
-            Order results by 'relevance', 'created_at', 'updated_at', or 'size'.
-            Defaults to 'relevance' if not specified.
-        order_direction : str, optional
-            Order direction, 'asc' for ascending or 'desc' for descending.
-            Defaults to 'desc'.
+        tags : List[str], optional
+            Filter results based on the 'tags' field. Documents matching any of the
+            provided tags will be returned.
+        path : str, optional
+            Filter results based on the 'path' field. Only documents whose path
+            starts with the provided value will be returned.
         nb_results : int, optional
             The number of results to return.
             Defaults to 50 if not specified.
@@ -225,6 +341,13 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
             List of public/authenticated documents the user has visited to limit
             the document returned to the ones the current user has seen.
             Built from linkreach list of a document in docs app.
+        search_type : str, optional
+            Type of search to perform: 'hybrid' or 'full_text'.
+            - 'hybrid': Uses hybrid search if enabled on the server,
+                otherwise falls back to full-text search.
+            - 'full_text': Uses only full-text search, even if hybrid search is enabled
+                on the server.
+            if the not specified, the server will use hybrid search when enabled
 
         Returns:
         --------
@@ -236,29 +359,37 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
         audience = self._get_service_provider_audience()
         user_sub = self.request.user.sub
         groups = []
-        # //////////////////////////////////////////////////
-
-        # Extract and validate query parameters using Pydantic schema
         params = schemas.SearchQueryParametersSchema(**request.data)
 
         # Get index list for search query
         try:
-            search_indices = self._get_opensearch_indices(
-                audience, services=params.services
-            )
+            search_indices = get_opensearch_indices(audience, services=params.services)
         except SuspiciousOperation as e:
-            return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST)
+            logger.error(e, exc_info=True)
+            return Response(
+                {"detail": "Invalid request."},
+                status=status.HTTP_400_BAD_REQUEST,
+            )
 
-        response = search(
+        logger.info("Search '%s' on indices %s", params.q, search_indices)
+        result = search(
             q=params.q,
             nb_results=params.nb_results,
-            order_by=params.order_by,
-            order_direction=params.order_direction,
             search_indices=search_indices,
             reach=params.reach,
             visited=params.visited,
             user_sub=user_sub,
             groups=groups,
-        )
+            tags=params.tags,
+            path=params.path,
+            search_type=params.search_type
+            if params.search_type
+            else SearchTypeEnum.HYBRID
+            if check_hybrid_search_enabled()
+            else SearchTypeEnum.FULL_TEXT,
+            rescore=params.rescore,
+        )["hits"]["hits"]
+        logger.info("found %d results", len(result))
+        logger.debug("results %s", result)
 
-        return Response(response["hits"]["hits"], status=status.HTTP_200_OK)
+        return Response(result, status=status.HTTP_200_OK)

src/backend/demo/defaults.py

@@ -13,4 +13,9 @@ DEV_SERVICES = (
         "client_id": "drive",
         "token": "find-api-key-for-driv-with-exactly-50-chars-length",
     },
+    {
+        "name": "conversations",
+        "client_id": "conversations",
+        "token": "find-api-key-for-conv-with-exactly-50-chars-length",
+    },
 )

src/backend/demo/management/commands/create_demo.py

@@ -15,7 +15,8 @@ from faker import Faker
 from opensearchpy.helpers import bulk
 
 from core import enums, factories
-from core.services.opensearch import ensure_index_exists, opensearch_client
+from core.services.indexing import ensure_index_exists
+from core.services.opensearch import opensearch_client
 
 from demo import defaults
 
@@ -127,8 +128,8 @@ def generate_document():
     )
 
     return {
-        "title": fake.sentence(nb_words=10, variable_nb_words=True),
-        "content": "\n".join(fake.paragraphs(nb=5)),
+        "title.en": fake.sentence(nb_words=10, variable_nb_words=True),
+        "content.en": "\n".join(fake.paragraphs(nb=5)),
         "created_at": created_at,
         "updated_at": updated_at,
         "size": random.randint(0, 100 * 1024**2),
@@ -143,7 +144,7 @@ def create_demo(stdout):
     Create a database with demo data for developers to work in a realistic environment.
     """
     opensearch_client_ = opensearch_client()
-    opensearch_client_.indices.delete("*")
+    opensearch_client_.indices.delete(index="*")
 
     with Timeit(stdout, "Creating services"):
         services = factories.ServiceFactory.create_batch(

src/backend/demo/tests/test_commands_create_demo.py

@@ -26,7 +26,7 @@ def test_commands_create_demo():
     """The create_demo management command should create objects as expected."""
     call_command("create_demo")
 
-    assert models.Service.objects.exclude(name="docs").count() == 3
+    assert models.Service.objects.exclude(name="docs").count() == 4
     assert opensearch_client().count()["count"] == 4
 
     docs = models.Service.objects.get(name="docs")

src/backend/evaluation/data/corpus/simple_corpus.py

@@ -0,0 +1,903 @@
+"""a simple corpus of documents for evaluation"""
+
+documents = [
+    {
+        "id": "sc-1",
+        "title": "La Révolution Française 1789",
+        "content": (
+            "La Révolution française débute le 14 juillet 1789 avec la prise de la "
+            "Bastille, symbole de l'absolutisme royal. Les causes sont multiples : crise "
+            "financière, famine, influence des Lumières et inégalités sociales criantes. "
+            "L'Assemblée Constituante abolit les privilèges le 4 août et adopte la "
+            "Déclaration des Droits de l'Homme le 26 août. Le roi Louis XVI est exécuté "
+            "en 1793 après la proclamation de la République. Cette révolution transforme "
+            "profondément la France et influence l'Europe entière."
+        ),
+        "updated_at": "2026-02-24T12:00:00Z",
+    },
+    {
+        "id": "sc-2",
+        "title": "La Première Guerre Mondiale",
+        "content": (
+            "La Première Guerre mondiale (1914-1918) oppose les Alliés (France, Royaume- "
+            "Uni, Russie) aux Empires centraux (Allemagne, Autriche-Hongrie). "
+            "L'assassinat de l'archiduc François-Ferdinand à Sarajevo déclenche le "
+            "conflit par le jeu des alliances. La guerre de tranchées caractérise le "
+            "front occidental avec Verdun et la Somme. Les nouvelles armes (gaz, tanks, "
+            "aviation) causent 18 millions de morts. Le traité de Versailles en 1919 "
+            "redessine la carte de l'Europe."
+        ),
+    },
+    {
+        "id": "sc-3",
+        "title": "La Décolonisation en Afrique",
+        "content": (
+            "La décolonisation de l'Afrique s'accélère après 1945, affaiblie par la "
+            "Seconde Guerre mondiale. Le Ghana obtient son indépendance en 1957, suivi de "
+            "17 pays en 1960, 'l'année de l'Afrique'. Le processus varie : négociation "
+            "pacifique (Afrique francophone) ou luttes armées (Algérie, Kenya). Les "
+            "frontières héritées de la colonisation créent des tensions ethniques. Les "
+            "nouveaux États font face à des défis économiques et politiques majeurs."
+        ),
+    },
+    {
+        "id": "sc-4",
+        "title": "Le Réchauffement Climatique",
+        "content": (
+            "Le réchauffement climatique désigne l'augmentation de la température moyenne "
+            "terrestre causée par les émissions de gaz à effet de serre. Depuis l'ère "
+            "industrielle, la température a augmenté de 1,2°C. Les conséquences incluent "
+            "fonte des glaciers, montée des eaux, événements météorologiques extrêmes et "
+            "migrations climatiques. Les accords de Paris en 2015 visent à limiter le "
+            "réchauffement à 1,5°C. La transition énergétique et les énergies "
+            "renouvelables sont essentielles."
+        ),
+    },
+    {
+        "id": "sc-5",
+        "title": "L'Empire Romain",
+        "content": (
+            "L'Empire romain domine le bassin méditerranéen pendant cinq siècles, de 27 "
+            "avant J.-C. à 476 après J.-C. Auguste, premier empereur, établit la Pax "
+            "Romana qui apporte prospérité et stabilité. Rome construit routes, aqueducs "
+            "et monuments grandioses comme le Colisée. Le christianisme devient religion "
+            "d'État en 380. Les invasions barbares et les crises internes provoquent la "
+            "chute de l'Empire d'Occident en 476."
+        ),
+    },
+    {
+        "id": "sc-6",
+        "title": "La Renaissance Européenne",
+        "content": (
+            "La Renaissance (XIVe-XVIe siècles) marque un renouveau artistique, "
+            "scientifique et intellectuel en Europe. Née en Italie avec Florence et les "
+            "Médicis, elle redécouvre l'Antiquité gréco-romaine. Léonard de Vinci, "
+            "Michel-Ange et Raphaël révolutionnent l'art. Gutenberg invente l'imprimerie "
+            "en 1450, facilitant la diffusion des savoirs. Les grandes découvertes "
+            "ouvrent le monde avec Colomb et Magellan. L'humanisme place l'homme au "
+            "centre de la réflexion."
+        ),
+    },
+    {
+        "id": "sc-7",
+        "title": "La Guerre Froide",
+        "content": (
+            "La Guerre Froide (1947-1991) oppose les États-Unis capitalistes à l'URSS "
+            "communiste sans conflit direct. Le rideau de fer divise l'Europe entre blocs "
+            "occidental et oriental. Les crises majeures incluent Berlin (1948, 1961), "
+            "Cuba (1962) et le Vietnam. La course aux armements nucléaires menace "
+            "l'humanité. La détente des années 1970 laisse place aux tensions des années "
+            "1980. La chute du mur de Berlin en 1989 symbolise la fin de la Guerre "
+            "Froide."
+        ),
+    },
+    {
+        "id": "sc-8",
+        "title": "L'Égypte Ancienne",
+        "content": (
+            "L'Égypte ancienne prospère pendant 3000 ans le long du Nil, de 3100 avant "
+            "J.-C. à 30 avant J.-C. Les pharaons sont considérés comme des dieux vivants. "
+            "Les pyramides de Gizeh, notamment celle de Khéops, témoignent de prouesses "
+            "architecturales. L'écriture hiéroglyphique permet l'administration et la "
+            "transmission des savoirs. Le culte des morts et la momification préparent à "
+            "la vie après la mort. Cléopâtre VII est la dernière souveraine avant la "
+            "conquête romaine."
+        ),
+    },
+    {
+        "id": "sc-9",
+        "title": "La Mondialisation",
+        "content": (
+            "La mondialisation désigne l'intégration croissante des économies et sociétés "
+            "à l'échelle planétaire. Accélérée depuis 1990, elle s'appuie sur les "
+            "nouvelles technologies, les transports et la libéralisation des échanges. "
+            "Les firmes multinationales organisent la production mondiale. Les flux "
+            "commerciaux, financiers et migratoires s'intensifient. Ce processus crée des "
+            "interdépendances mais accentue aussi les inégalités entre pays développés et "
+            "en développement."
+        ),
+    },
+    {
+        "id": "sc-10",
+        "title": "La Conquête Spatiale",
+        "content": (
+            "La conquête spatiale débute en 1957 avec le satellite soviétique Spoutnik. "
+            "Youri Gagarine devient le premier homme dans l'espace en 1961. Les "
+            "Américains répondent avec le programme Apollo : Neil Armstrong marche sur la "
+            "Lune en 1969. La Station Spatiale Internationale symbolise la coopération "
+            "internationale depuis 1998. Aujourd'hui, les ambitions incluent Mars, et des "
+            "entreprises privées comme SpaceX révolutionnent l'accès à l'espace."
+        ),
+    },
+    {
+        "id": "sc-11",
+        "title": "Les Métropoles Mondiales",
+        "content": (
+            "Les métropoles mondiales comme New York, Londres, Tokyo et Paris concentrent "
+            "pouvoir économique, politique et culturel. Elles abritent sièges sociaux des "
+            "multinationales, bourses financières et institutions internationales. Ces "
+            "villes sont des nœuds de transport et communication globaux. Elles attirent "
+            "flux migratoires et talents créant diversité mais aussi gentrification. Les "
+            "mégapoles du Sud (Mumbai, São Paulo) connaissent une urbanisation rapide "
+            "avec des défis d'infrastructures."
+        ),
+    },
+    {
+        "id": "sc-12",
+        "title": "La Seconde Guerre Mondiale",
+        "content": (
+            "La Seconde Guerre mondiale (1939-1945) est le conflit le plus meurtrier de "
+            "l'histoire avec 60 millions de morts. Hitler envahit la Pologne en septembre "
+            "1939, déclenchant la guerre. La France capitule en 1940 tandis que le "
+            "Royaume-Uni résiste. L'Allemagne attaque l'URSS en 1941 et le Japon Pearl "
+            "Harbor. Le débarquement en Normandie en 1944 libère l'Europe. La Shoah "
+            "extermine 6 millions de Juifs. Le conflit se termine avec les bombes "
+            "atomiques sur Hiroshima et Nagasaki."
+        ),
+    },
+    {
+        "id": "sc-13",
+        "title": "Le Développement Durable",
+        "content": (
+            "Le développement durable vise à concilier croissance économique, protection "
+            "environnementale et équité sociale. Défini au Sommet de Rio en 1992, il "
+            "répond aux besoins présents sans compromettre ceux des générations futures. "
+            "Les 17 Objectifs de Développement Durable de l'ONU incluent éradication de "
+            "la pauvreté, éducation, égalité des genres et action climatique. Les "
+            "énergies renouvelables, l'économie circulaire et la consommation responsable "
+            "sont des leviers essentiels."
+        ),
+    },
+    {
+        "id": "sc-14",
+        "title": "La Chine contemporaine",
+        "content": (
+            "La Chine est devenue la deuxième économie mondiale grâce aux réformes de "
+            "Deng Xiaoping en 1978. L'ouverture au capitalisme maintient le régime "
+            "communiste du Parti unique. Le pays est 'l'usine du monde' avec des zones "
+            "économiques spéciales. Les nouvelles routes de la soie étendent son "
+            "influence internationale. Avec 1,4 milliard d'habitants, la Chine fait face "
+            "à des défis environnementaux, démographiques et sociaux majeurs."
+        ),
+    },
+    {
+        "id": "sc-15",
+        "title": "Les Grandes Découvertes",
+        "content": (
+            "Les grandes découvertes (XVe-XVIe siècles) voient les Européens explorer le "
+            "monde. Christophe Colomb atteint l'Amérique en 1492, croyant trouver les "
+            "Indes. Vasco de Gama ouvre la route des Indes en 1498. Magellan réalise le "
+            "premier tour du monde en 1522. Ces expéditions sont motivées par la "
+            "recherche d'épices, d'or et l'évangélisation. Elles inaugurent la "
+            "colonisation européenne et l'échange colombien qui transforment les "
+            "civilisations."
+        ),
+    },
+    {
+        "id": "sc-16",
+        "title": "L'Union Européenne",
+        "content": (
+            "L'Union européenne naît du désir de paix après 1945. Le traité de Rome en "
+            "1957 crée la CEE avec six membres fondateurs. L'UE compte aujourd'hui 27 "
+            "États après le Brexit. Le marché unique permet libre circulation des biens, "
+            "services, capitaux et personnes. L'euro est adopté par 20 pays. L'UE fait "
+            "face à des défis : crise migratoire, euroscepticisme et divergences "
+            "économiques entre Nord et Sud."
+        ),
+    },
+    {
+        "id": "sc-17",
+        "title": "La Révolution Industrielle",
+        "content": (
+            "La révolution industrielle débute en Angleterre au XVIIIe siècle avec la "
+            "machine à vapeur de Watt. L'industrie textile mécanisée précède la "
+            "métallurgie et les chemins de fer. Le charbon et la vapeur fournissent "
+            "l'énergie nécessaire. L'urbanisation s'accélère avec l'exode rural vers les "
+            "usines. Les conditions ouvrières misérables suscitent les mouvements sociaux "
+            "et le syndicalisme. Cette révolution transforme radicalement l'économie et "
+            "la société européennes."
+        ),
+    },
+    {
+        "id": "sc-18",
+        "title": "Les Inégalités Nord-Sud",
+        "content": (
+            "Les inégalités Nord-Sud opposent pays développés et pays en développement. "
+            "L'Indice de Développement Humain mesure richesse, éducation et santé. Les "
+            "pays du Nord concentrent richesses et technologies tandis que le Sud subit "
+            "pauvreté et dépendance économique. L'héritage colonial, l'endettement et les "
+            "termes de l'échange inégaux perpétuent ces écarts. Les pays émergents "
+            "(BRICS) remettent en question cette division binaire."
+        ),
+    },
+    {
+        "id": "sc-19",
+        "title": "Le Siècle des Lumières",
+        "content": (
+            "Le Siècle des Lumières (XVIIIe siècle) est un mouvement intellectuel "
+            "européen promouvant raison, science et progrès. Voltaire critique "
+            "l'intolérance religieuse, Montesquieu théorise la séparation des pouvoirs, "
+            "Rousseau défend la souveraineté populaire. L'Encyclopédie de Diderot et "
+            "d'Alembert diffuse les savoirs. Ces philosophes remettent en cause "
+            "l'absolutisme et les privilèges. Leurs idées inspirent les révolutions "
+            "américaine et française."
+        ),
+    },
+    {
+        "id": "sc-20",
+        "title": "Les Ressources Énergétiques",
+        "content": (
+            "Les ressources énergétiques sont inégalement réparties sur la planète. Les "
+            "énergies fossiles (pétrole, gaz, charbon) dominent mais sont non "
+            "renouvelables et polluantes. Le Moyen-Orient concentre 48% des réserves "
+            "pétrolières mondiales. La Russie utilise le gaz comme arme géopolitique. Les "
+            "énergies renouvelables (solaire, éolien, hydraulique) progressent face au "
+            "réchauffement climatique. La transition énergétique est un défi majeur du "
+            "XXIe siècle pour assurer souveraineté et durabilité."
+        ),
+    },
+    {
+        "id": "sc-21",
+        "title": "Le Lion d'Afrique",
+        "content": (
+            "Le lion est le plus grand félin d'Afrique et vit en groupe social appelé "
+            "troupe. Les mâles se distinguent par leur majestueuse crinière qui protège "
+            "leur cou lors des combats. Les lions chassent principalement au crépuscule, "
+            "les lionnes étant les principales chasseuses. Ils peuvent rugir jusqu'à 8 "
+            "kilomètres de distance pour marquer leur territoire. Malheureusement, leur "
+            "population a diminué de 43% en 20 ans."
+        ),
+    },
+    {
+        "id": "sc-22",
+        "title": "Le Dauphin",
+        "content": (
+            "Les dauphins sont parmi les animaux les plus intelligents de la planète. Ils "
+            "communiquent par des clics et sifflements complexes, chaque individu "
+            "possédant sa propre signature sonore. Vivant en groupes sociaux "
+            "sophistiqués, ils chassent en coordination et s'entraident. Les dauphins "
+            "peuvent nager jusqu'à 30 km/h et plonger à 300 mètres de profondeur. Leur "
+            "cerveau possède plus de circonvolutions que celui de l'homme."
+        ),
+    },
+    {
+        "id": "sc-23",
+        "title": "L'Éléphant d'Asie",
+        "content": (
+            "L'éléphant d'Asie est légèrement plus petit que son cousin africain mais "
+            "tout aussi majestueux. Ces herbivores peuvent consommer jusqu'à 150 kg de "
+            "végétation par jour et boire 140 litres d'eau. Leur trompe contient 40 000 "
+            "muscles et leur sert à manger, boire, se laver et communiquer. Les éléphants "
+            "vivent en matriarcat dirigé par la femelle la plus âgée. Leur mémoire "
+            "exceptionnelle leur permet de se souvenir des points d'eau sur de vastes "
+            "territoires."
+        ),
+    },
+    {
+        "id": "sc-24",
+        "title": "Le Colibri",
+        "content": (
+            "Le colibri est le plus petit oiseau du monde, certaines espèces ne pesant "
+            "que 2 grammes. Ses ailes battent jusqu'à 80 fois par seconde, lui permettant "
+            "de faire du vol stationnaire et même de voler en arrière. Son métabolisme "
+            "est si rapide qu'il doit manger jusqu'à deux fois son poids en nectar "
+            "quotidiennement. Le colibri peut visiter 1000 fleurs par jour. Ses couleurs "
+            "iridescentes changent selon l'angle de la lumière."
+        ),
+    },
+    {
+        "id": "sc-25",
+        "title": "L'Ours polaire",
+        "content": (
+            "L'ours polaire est le plus grand carnivore terrestre, pouvant peser jusqu'à "
+            "800 kg. Sa fourrure blanche et sa peau noire lui permettent d'absorber la "
+            "chaleur du soleil. Son odorat est si développé qu'il peut détecter un phoque "
+            "sous 1 mètre de glace à 1,5 km. Le réchauffement climatique menace gravement "
+            "son habitat."
+        ),
+    },
+    {
+        "id": "sc-26",
+        "title": "Le Caméléon",
+        "content": (
+            "Le caméléon possède la capacité extraordinaire de changer de couleur selon "
+            "son humeur, la température et la communication sociale. Ses yeux peuvent "
+            "bouger indépendamment l'un de l'autre à 360 degrés. Sa langue peut s'étendre "
+            "jusqu'à deux fois la longueur de son corps pour capturer des insectes. "
+            "Certaines espèces mesurent seulement 3 cm tandis que d'autres atteignent 70 "
+            "cm. Ils se déplacent lentement et de manière saccadée pour imiter le "
+            "mouvement des feuilles."
+        ),
+    },
+    {
+        "id": "sc-27",
+        "title": "Le Manchot Empereur",
+        "content": (
+            "Le manchot empereur survit aux conditions les plus extrêmes de la planète en "
+            "Antarctique. Les mâles couvent l'œuf unique pendant 64 jours dans des "
+            "températures atteignant -40°C sans manger. Ils se regroupent en tortue pour "
+            "se protéger du froid, tournant régulièrement pour partager la chaleur. Ces "
+            "oiseaux peuvent plonger jusqu'à 500 mètres de profondeur et retenir leur "
+            "respiration 22 minutes. Leur parade nuptiale comprend des chants complexes "
+            "et synchronisés."
+        ),
+    },
+    {
+        "id": "sc-28",
+        "title": "Le Requin Blanc",
+        "content": (
+            "Le grand requin blanc est l'un des prédateurs marins les plus redoutés et "
+            "fascinants. Il peut atteindre 6 mètres de long et peser plus de 2 tonnes. "
+            "Ses dents triangulaires et dentelées se renouvellent constamment tout au "
+            "long de sa vie. Il peut détecter une goutte de sang dans 100 litres d'eau "
+            "grâce à son odorat exceptionnel. Contrairement à sa réputation, il attaque "
+            "rarement l'homme, préférant les phoques et les otaries."
+        ),
+    },
+    {
+        "id": "sc-29",
+        "title": "L'Abeille",
+        "content": (
+            "Les abeilles jouent un rôle crucial dans la pollinisation de 80% des plantes "
+            "à fleurs. Une ruche peut contenir jusqu'à 60 000 abeilles organisées en "
+            "société matriarcale autour de la reine. Les ouvrières communiquent la "
+            "localisation des fleurs par une danse complexe en forme de huit. Une abeille "
+            "produit environ 1/12 de cuillère à café de miel dans sa vie. Leur déclin "
+            "inquiétant menace notre sécurité alimentaire."
+        ),
+    },
+    {
+        "id": "sc-30",
+        "title": "Le Guépard",
+        "content": (
+            "Le guépard est l'animal terrestre le plus rapide, capable d'atteindre 110 "
+            "km/h en quelques secondes. Contrairement aux autres félins, il ne peut pas "
+            "rétracter complètement ses griffes, ce qui lui donne une meilleure "
+            "adhérence. Sa queue longue lui sert de balancier lors des changements de "
+            "direction. Son corps élancé et ses poumons surdimensionnés sont optimisés "
+            "pour la course. Il chasse le jour pour éviter les autres prédateurs plus "
+            "puissants."
+        ),
+    },
+    {
+        "id": "sc-31",
+        "title": "La Pieuvre",
+        "content": (
+            "La pieuvre possède trois cœurs et un sang bleu à base de cuivre. Son "
+            "intelligence remarquable lui permet de résoudre des problèmes complexes et "
+            "d'utiliser des outils. Chacun de ses huit bras contient des neurones, lui "
+            "donnant une forme de pensée décentralisée. Elle peut changer de couleur et "
+            "de texture instantanément pour se camoufler. Certaines espèces peuvent "
+            "passer à travers des ouvertures de la taille d'une pièce de monnaie."
+        ),
+    },
+    {
+        "id": "sc-32",
+        "title": "Le Koala",
+        "content": (
+            "Le koala dort jusqu'à 20 heures par jour pour économiser l'énergie "
+            "nécessaire à digérer l'eucalyptus toxique. Il se nourrit exclusivement de "
+            "feuilles d'eucalyptus, pouvant distinguer entre 600 espèces différentes. Son "
+            "système digestif unique contient des bactéries spéciales pour neutraliser "
+            "les toxines. Les bébés koalas naissent de la taille d'un haricot et restent "
+            "6 mois dans la poche maternelle. Ils ne boivent presque jamais, tirant l'eau "
+            "des feuilles qu'ils consomment."
+        ),
+    },
+    {
+        "id": "sc-33",
+        "title": "Le Gorille des Montagnes",
+        "content": (
+            "Le gorille des montagnes est l'un de nos plus proches parents, partageant "
+            "98% de notre ADN. Les mâles silverback peuvent peser jusqu'à 200 kg et sont "
+            "d'une force exceptionnelle. Malgré leur puissance, ce sont des animaux "
+            "pacifiques et végétariens. Ils vivent en groupes familiaux dirigés par un "
+            "mâle dominant et communiquent par 25 vocalisations différentes. Moins de "
+            "1000 individus survivent dans les forêts d'Afrique centrale."
+        ),
+    },
+    {
+        "id": "sc-34",
+        "title": "Le Papillon Monarque",
+        "content": (
+            "Le papillon monarque effectue une migration annuelle de 4000 km entre le "
+            "Canada et le Mexique. Aucun individu ne complète le voyage entier, il faut 4 "
+            "générations pour accomplir le cycle. Ils utilisent le soleil comme boussole "
+            "et possèdent une horloge circadienne sophistiquée. Leur couleur orange vif "
+            "avertit les prédateurs de leur toxicité acquise en mangeant de l'asclépiade. "
+            "Des millions de papillons se regroupent dans quelques forêts mexicaines en "
+            "hiver."
+        ),
+    },
+    {
+        "id": "sc-35",
+        "title": "Le Loup Gris",
+        "content": (
+            "Le loup gris vit en meute organisée hiérarchiquement autour d'un couple "
+            "alpha. Ces chasseurs coopératifs peuvent traquer des proies bien plus "
+            "grandes qu'eux grâce à leur coordination. Ils communiquent par hurlements "
+            "pouvant porter jusqu'à 10 km dans les forêts. Un loup peut parcourir 70 km "
+            "en une nuit à la recherche de nourriture. Leur réintroduction dans certains "
+            "écosystèmes a démontré leur rôle crucial dans l'équilibre naturel."
+        ),
+    },
+    {
+        "id": "sc-36",
+        "title": "Le Kangourou Roux",
+        "content": (
+            "Le kangourou roux est le plus grand marsupial au monde, pouvant atteindre 2 "
+            "mètres de haut. Il peut sauter jusqu'à 9 mètres de long et maintenir une "
+            "vitesse de 50 km/h. Sa queue musclée lui sert de trépied au repos et de "
+            "balancier en mouvement. Les femelles peuvent retarder le développement d'un "
+            "embryon si les conditions sont défavorables. Parfaitement adapté au climat "
+            "aride australien, il peut passer des mois sans boire."
+        ),
+    },
+    {
+        "id": "sc-37",
+        "title": "La Baleine à Bosse",
+        "content": (
+            "Les baleines à bosse produisent des chants complexes pouvant durer 20 "
+            "minutes et s'entendre à des centaines de kilomètres. Les mâles d'une même "
+            "région partagent le même chant qui évolue chaque année. Malgré leur masse de "
+            "30 tonnes, elles peuvent sauter entièrement hors de l'eau. Elles migrent sur "
+            "25 000 km par an entre zones d'alimentation et de reproduction. Leurs "
+            "nageoires pectorales sont les plus longues de tous les cétacés."
+        ),
+    },
+    {
+        "id": "sc-38",
+        "title": "Le Serpent Python",
+        "content": (
+            "Le python réticulé peut atteindre 9 mètres de long, ce qui en fait l'un des "
+            "plus longs serpents. Il tue sa proie par constriction, resserrant son "
+            "étreinte à chaque expiration de la victime. Ces serpents peuvent passer des "
+            "mois sans manger après avoir avalé une grande proie. Leurs organes "
+            "sensoriels thermiques leur permettent de détecter la chaleur corporelle dans "
+            "l'obscurité. Contrairement aux idées reçues, ils ne peuvent pas avaler un "
+            "humain adulte."
+        ),
+    },
+    {
+        "id": "sc-39",
+        "title": "Le Hibou Grand-Duc",
+        "content": (
+            "Le hibou grand-duc est le plus grand rapace nocturne d'Europe avec une "
+            "envergure de 2 mètres. Son vol silencieux est rendu possible par la "
+            "structure spéciale de ses plumes. Il peut tourner sa tête à 270 degrés grâce "
+            "à 14 vertèbres cervicales. Sa vision nocturne est 100 fois plus sensible que "
+            "celle de l'humain. Son hululement puissant peut s'entendre jusqu'à 2 km. Il "
+            "chasse des proies variées, du mulot au renardeau."
+        ),
+    },
+    {
+        "id": "sc-40",
+        "title": "Le Paresseux",
+        "content": (
+            "Le paresseux se déplace si lentement que des algues poussent sur sa "
+            "fourrure, lui donnant un camouflage verdâtre. Il descend de son arbre une "
+            "fois par semaine pour déféquer, moment où il est le plus vulnérable. Son "
+            "métabolisme est le plus lent de tous les non-hibernants. Il peut tourner sa "
+            "tête à 270 degrés pour surveiller les prédateurs. Les paresseux passent 90% "
+            "de leur vie suspendus la tête en bas et peuvent même dormir ainsi."
+        ),
+    },
+    {
+        "id": "sc-41",
+        "title": "Bœuf Bourguignon",
+        "content": (
+            "Le bœuf bourguignon est un plat mijoté emblématique de la cuisine française. "
+            "Coupez 1,5 kg de bœuf en cubes et faites-les revenir dans du beurre. Ajoutez "
+            "2 carottes, 2 oignons, 3 gousses d'ail et un bouquet garni. Versez 75 cl de "
+            "vin rouge de Bourgogne et 25 cl de bouillon. Laissez mijoter 3 heures à feu "
+            "doux. Ajoutez 200g de champignons et 150g de lardons dorés. Servez avec des "
+            "pommes de terre vapeur ou des tagliatelles fraîches."
+        ),
+    },
+    {
+        "id": "sc-42",
+        "title": "Ratatouille Provençale",
+        "content": (
+            "Tranchez finement 2 aubergines, 2 courgettes, 2 poivrons, 4 tomates et 1 "
+            "oignon. Dans une cocotte, faites revenir l'oignon et l'ail dans de l'huile "
+            "d'olive. Ajoutez les légumes par couches en alternant. Assaisonnez avec du "
+            "thym, du romarin, sel et poivre. Couvrez et laissez mijoter 40 minutes à feu "
+            "doux. La ratatouille peut se déguster chaude ou froide, accompagnée de riz "
+            "ou de pain grillé."
+        ),
+    },
+    {
+        "id": "sc-43",
+        "title": "Coq au Vin",
+        "content": (
+            "Découpez un coq ou un poulet fermier en morceaux. Faites mariner 12 heures "
+            "dans 75 cl de vin rouge avec carottes, oignons et aromates. Égouttez et "
+            "faites dorer les morceaux avec 100g de lardons. Flambez au cognac puis "
+            "ajoutez la marinade filtrée. Laissez mijoter 1h30. Ajoutez des champignons "
+            "et des petits oignons grelots. Liez la sauce avec du beurre manié. Servez "
+            "avec des croûtons aillés."
+        ),
+    },
+    {
+        "id": "sc-44",
+        "title": "Tarte Tatin",
+        "content": (
+            "Dans un moule à tarte, faites fondre 100g de beurre avec 100g de sucre pour "
+            "créer un caramel. Disposez 6 pommes Reinette coupées en quartiers en rosace "
+            "serrée. Enfournez 30 minutes à 180°C. Recouvrez de pâte feuilletée et "
+            "enfournez 25 minutes supplémentaires. Laissez tiédir 10 minutes avant de "
+            "retourner sur un plat. Servez tiède avec de la crème fraîche ou une boule de "
+            "glace vanille."
+        ),
+    },
+    {
+        "id": "sc-45",
+        "title": "Bouillabaisse Marseillaise",
+        "content": (
+            "Faites suer dans l'huile d'olive 2 oignons, 4 tomates, 4 gousses d'ail et du "
+            "fenouil. Ajoutez safran, thym, laurier et zeste d'orange. Versez 2 litres de "
+            "fumet de poisson et portez à ébullition. Ajoutez 2 kg de poissons variés "
+            "(rascasse, grondin, rouget) coupés en morceaux. Cuisez 15 minutes à gros "
+            "bouillons. Servez avec des croûtons, de la rouille et du gruyère râpé."
+        ),
+    },
+    {
+        "id": "sc-46",
+        "title": "Quiche Lorraine",
+        "content": (
+            "Garnissez un moule de pâte brisée. Faites revenir 200g de lardons fumés sans "
+            "matière grasse. Battez 4 œufs avec 30 cl de crème fraîche, sel, poivre et "
+            "noix de muscade. Disposez les lardons sur la pâte et versez l'appareil. "
+            "Ajoutez éventuellement 100g de gruyère râpé. Enfournez 35 minutes à 180°C "
+            "jusqu'à ce que la garniture soit dorée et gonflée. Servez tiède avec une "
+            "salade verte."
+        ),
+    },
+    {
+        "id": "sc-47",
+        "title": "Blanquette de Veau",
+        "content": (
+            "Coupez 1,2 kg d'épaule de veau en cubes. Blanchissez-les 5 minutes à l'eau "
+            "bouillante puis rincez. Remettez dans une cocotte avec carottes, oignons "
+            "piqués de clous de girofle et bouquet garni. Couvrez d'eau et laissez "
+            "mijoter 1h30. Préparez un roux avec beurre et farine, ajoutez le bouillon "
+            "filtré. Liez avec 2 jaunes d'œufs et 20 cl de crème. Ajoutez champignons et "
+            "oignons grelots. Servez avec du riz."
+        ),
+    },
+    {
+        "id": "sc-48",
+        "title": "Crêpes Bretonnes",
+        "content": (
+            "Mélangez 250g de farine, 4 œufs, 50 cl de lait et une pincée de sel. Ajoutez "
+            "50g de beurre fondu et laissez reposer 1 heure. Huilez légèrement une poêle "
+            "chaude et versez une louche de pâte. Faites cuire 2 minutes de chaque côté "
+            "jusqu'à ce que la crêpe soit dorée. Garnissez de sucre, confiture, chocolat "
+            "ou jambon-fromage selon vos envies. Servez immédiatement."
+        ),
+    },
+    {
+        "id": "sc-49",
+        "title": "Cassoulet Toulousain",
+        "content": (
+            "Faites tremper 500g de haricots blancs 12 heures. Cuisez-les 1 heure avec "
+            "carottes, oignons et bouquet garni. Faites revenir 4 cuisses de canard "
+            "confites, 400g de saucisse de Toulouse et 200g de poitrine fumée. Mélangez "
+            "haricots et viandes dans une cocotte avec la graisse de canard. Couvrez de "
+            "chapelure et enfournez 1 heure à 160°C. Cassez la croûte plusieurs fois "
+            "pendant la cuisson."
+        ),
+    },
+    {
+        "id": "sc-50",
+        "title": "Soufflé au Fromage",
+        "content": (
+            "Préparez une béchamel avec 40g de beurre, 40g de farine et 25 cl de lait. "
+            "Hors du feu, incorporez 150g de gruyère râpé et 4 jaunes d'œufs. Montez 5 "
+            "blancs en neige ferme avec une pincée de sel. Incorporez délicatement les "
+            "blancs à la préparation. Versez dans des ramequins beurrés et farinés. "
+            "Enfournez 20 minutes à 180°C sans ouvrir le four. Servez immédiatement."
+        ),
+    },
+    {
+        "id": "sc-51",
+        "title": "Pot-au-feu",
+        "content": (
+            "Dans une grande marmite, placez 1,5 kg de viande de bœuf (gîte, paleron, "
+            "plat de côtes). Couvrez d'eau froide et portez à ébullition en écumant. "
+            "Ajoutez 4 carottes, 4 poireaux, 2 navets, 2 oignons piqués de clous de "
+            "girofle et un bouquet garni. Laissez mijoter 3 heures à feu doux. Servez le "
+            "bouillon en entrée puis la viande et les légumes avec cornichons, moutarde "
+            "et gros sel."
+        ),
+    },
+    {
+        "id": "sc-52",
+        "title": "Mousse au Chocolat",
+        "content": (
+            "Faites fondre 200g de chocolat noir au bain-marie. Séparez 6 œufs et "
+            "incorporez les jaunes au chocolat fondu tiède. Montez les blancs en neige "
+            "ferme avec une pincée de sel et 20g de sucre. Incorporez délicatement les "
+            "blancs au chocolat en trois fois avec une spatule. Répartissez dans des "
+            "verrines et réfrigérez 4 heures minimum. Décorez de copeaux de chocolat ou "
+            "de chantilly."
+        ),
+    },
+    {
+        "id": "sc-53",
+        "title": "Gratin Dauphinois",
+        "content": (
+            "Épluchez et émincez finement 1,5 kg de pommes de terre. Frottez un plat à "
+            "gratin avec une gousse d'ail. Disposez les pommes de terre en couches en "
+            "assaisonnant de sel, poivre et noix de muscade. Mélangez 50 cl de crème avec "
+            "25 cl de lait et versez sur les pommes de terre. Ajoutez quelques noisettes "
+            "de beurre. Enfournez 1h15 à 160°C jusqu'à obtenir une croûte dorée."
+        ),
+    },
+    {
+        "id": "sc-54",
+        "title": "Salade de légumes",
+        "content": (
+            "Disposez sur un plat 4 tomates en quartiers, 1 concombre émincé, 1 poivron "
+            "coupé, 200g de haricots verts cuits, 4 œufs durs, 100g d'olives noires et 8 "
+            "filets d'anchois. Préparez une vinaigrette avec huile d'olive, vinaigre de "
+            "vin, moutarde, ail et basilic. Arrosez généreusement et servez frais. "
+            "Certains ajoutent des artichauts ou des fèves."
+        ),
+    },
+    {
+        "id": "sc-55",
+        "title": "Tarte au Citron Meringuée",
+        "content": (
+            "Garnissez un moule de pâte sablée et faites-la cuire à blanc 15 minutes. "
+            "Préparez une crème avec 4 jaunes d'œufs, 150g de sucre, le zeste et jus de 3 "
+            "citrons. Cuisez à feu doux en remuant jusqu'à épaississement. Versez sur la "
+            "pâte. Montez 4 blancs en neige avec 100g de sucre. Couvrez la tarte de "
+            "meringue en formant des pointes. Enfournez 10 minutes à 150°C pour dorer."
+        ),
+    },
+    {
+        "id": "sc-56",
+        "title": "Magret de Canard aux Figues",
+        "content": (
+            "Quadrillez la peau de 2 magrets sans entamer la chair. Salez et poivrez. "
+            "Posez-les côté peau dans une poêle froide. Faites cuire 7 minutes puis "
+            "retournez pour 4 minutes (rosé). Réservez au chaud. Dans la graisse, faites "
+            "revenir 8 figues coupées en deux avec 2 cuillères de miel et un trait de "
+            "vinaigre balsamique. Tranchez les magrets et nappez de sauce aux figues."
+        ),
+    },
+    {
+        "id": "sc-57",
+        "title": "Clafoutis aux Cerises",
+        "content": (
+            "Disposez 500g de cerises lavées et équeutées (traditionnellement non "
+            "dénoyautées) dans un plat beurré. Battez 4 œufs avec 100g de sucre jusqu'à "
+            "ce que le mélange blanchisse. Ajoutez 100g de farine, 50 cl de lait et une "
+            "pincée de sel. Versez l'appareil sur les cerises. Enfournez 45 minutes à "
+            "180°C. Saupoudrez de sucre glace et servez tiède ou froid."
+        ),
+    },
+    {
+        "id": "sc-58",
+        "title": "Fondue Savoyarde",
+        "content": (
+            "Frottez un caquelon avec une gousse d'ail coupée. Versez 15 cl de vin blanc "
+            "sec et faites chauffer. Ajoutez 400g de comté, 400g de beaufort et 200g de "
+            "reblochon râpés. Remuez en forme de 8 jusqu'à ce que le fromage fonde. "
+            "Ajoutez 5 cl de kirsch et du poivre. Maintenez au chaud sur un réchaud. "
+            "Trempez des cubes de pain rassis piqués sur des fourchettes."
+        ),
+    },
+    {
+        "id": "sc-59",
+        "title": "Crème Brûlée",
+        "content": (
+            "Faites chauffer 50 cl de crème avec une gousse de vanille fendue. Battez 6 "
+            "jaunes d'œufs avec 100g de sucre. Versez la crème chaude en remuant puis "
+            "filtrez. Répartissez dans des ramequins. Cuisez au bain-marie 40 minutes à "
+            "150°C. Réfrigérez 4 heures. Saupoudrez généreusement de sucre et caramélisez "
+            "au chalumeau ou sous le gril. Laissez durcir 2 minutes avant de servir."
+        ),
+    },
+    {
+        "id": "sc-60",
+        "title": "Tartare de Bœuf",
+        "content": (
+            "Hachez finement 600g de filet de bœuf bien frais au couteau. Ajoutez 2 "
+            "échalotes ciselées, 2 cuillères de câpres, 4 cornichons hachés, persil et "
+            "ciboulette. Assaisonnez avec moutarde, sauce Worcestershire, Tabasco, sel et "
+            "poivre. Incorporez 2 jaunes d'œufs et un filet d'huile d'olive. Formez des "
+            "dômes et servez avec des frites maison et une salade verte. Se prépare au "
+            "dernier moment."
+        ),
+    },
+    {
+        "id": "sc-61",
+        "title": "La Musique",
+        "content": (
+            "La musique est l'art d'organiser les sons dans le temps selon le rythme, la "
+            "mélodie et l'harmonie. Présente dans toutes les cultures, elle accompagne "
+            "les rites, les émotions et les récits humains depuis la préhistoire. Des "
+            "modes antiques grecs au contrepoint baroque, du jazz à l'électro, elle "
+            "évolue sans cesse. Les compositeurs explorent les timbres et les structures "
+            "tandis que l’improvisation garde la spontanéité vivante. La musique relie "
+            "mathématique, émotion et mouvement."
+        ),
+    },
+    {
+        "id": "sc-62",
+        "title": "Le Cinéma",
+        "content": (
+            "Le cinéma est l'art de raconter des histoires par le mouvement des images et "
+            "le son. Né à la fin du XIXe siècle avec les frères Lumière, il a rapidement "
+            "fusionné technique et poésie. Le montage, la lumière et la mise en scène en "
+            "font un art total mêlant littérature, théâtre et musique. Du muet de Chaplin "
+            "au cinéma numérique, chaque époque invente un nouveau langage visuel. Le "
+            "cinéma explore la mémoire, les rêves et la condition humaine à travers "
+            "l’écran."
+        ),
+    },
+    {
+        "id": "sc-63",
+        "title": "La Danse",
+        "content": (
+            "La danse est l’art du mouvement du corps dans l’espace et le temps, souvent "
+            "accompagné de musique. Elle exprime des émotions, raconte des histoires ou "
+            "célèbre des rites. Des danses tribales aux ballets classiques, des danses "
+            "contemporaines au hip-hop, chaque culture invente ses gestes et son rythme. "
+            "La chorégraphie unit discipline et liberté, le corps devenant un instrument "
+            "expressif. La danse relie énergie, esthétique et communication non verbale."
+        ),
+    },
+    {
+        "id": "sc-64",
+        "title": "La Peinture à l'Huile",
+        "content": (
+            "La peinture à l'huile est une technique artistique utilisant des pigments "
+            "mélangés à de l'huile siccative, généralement de lin. Inventée au XVe siècle "
+            "et perfectionnée par les maîtres flamands comme Van Eyck, elle permet des "
+            "glacis subtils et des dégradés lumineux. Le temps de séchage lent offre la "
+            "possibilité de travailler les transitions et les détails. Les grands maîtres "
+            "comme Rembrandt, Vermeer et plus tard les impressionnistes ont exploité ses "
+            "possibilités. Cette technique reste aujourd'hui la plus prisée pour la "
+            "peinture de chevalet."
+        ),
+    },
+    {
+        "id": "sc-65",
+        "title": "La Sculpture sur Pierre",
+        "content": (
+            "La sculpture sur pierre est l'un des arts les plus anciens de l'humanité, "
+            "remontant à la préhistoire. Le sculpteur taille directement dans le marbre, "
+            "le granit ou le calcaire avec des ciseaux et des masses. Michel-Ange "
+            "considérait que la statue existait déjà dans le bloc, il suffisait de "
+            "libérer la forme. Cette technique soustractive ne pardonne pas l'erreur. Les "
+            "œuvres comme le David ou la Pietà démontrent la capacité de donner vie et "
+            "émotion à la pierre froide."
+        ),
+    },
+    {
+        "id": "sc-66",
+        "title": "La Calligraphie",
+        "content": (
+            "La calligraphie est l'art de former les lettres avec beauté et harmonie. En "
+            "Occident, les moines copistes médiévaux ont perfectionné l'onciale et la "
+            "gothique. En Asie, la calligraphie chinoise et japonaise est considérée "
+            "comme la forme d'art la plus pure, où chaque trait exprime l'énergie et "
+            "l'esprit de l'artiste. L'outil traditionnel est le pinceau ou le calame. La "
+            "maîtrise nécessite des années de pratique pour contrôler la pression, la "
+            "vitesse et le rythme."
+        ),
+    },
+    {
+        "id": "sc-67",
+        "title": "La Photographie",
+        "content": (
+            "La photographie transforme la capture d'images en expression créative depuis "
+            "le XIXe siècle. Des pionniers comme Ansel Adams et Henri Cartier-Bresson ont "
+            "élevé le médium au rang d'art majeur. La composition, la lumière, le cadrage "
+            "et le moment décisif sont essentiels. Le passage au numérique a ouvert de "
+            "nouvelles possibilités de post-traitement. La photographie d'art explore "
+            "tous les genres : portrait, paysage, abstrait, documentaire et conceptuel."
+        ),
+    },
+    {
+        "id": "sc-68",
+        "title": "La Danse Contemporaine",
+        "content": (
+            "La danse contemporaine émerge au XXe siècle comme rupture avec le ballet "
+            "classique. Des chorégraphes comme Martha Graham, Merce Cunningham et Pina "
+            "Bausch explorent de nouveaux langages corporels. Cette forme privilégie "
+            "l'expression émotionnelle, la liberté de mouvement et l'improvisation. Le "
+            "corps devient un outil de questionnement social et politique. Les spectacles "
+            "intègrent souvent d'autres disciplines comme la vidéo, le théâtre et la "
+            "musique expérimentale."
+        ),
+    },
+    {
+        "id": "sc-69",
+        "title": "L'Origami",
+        "content": (
+            "L'origami est l'art japonais du pliage de papier, transformant une feuille "
+            "plane en sculpture tridimensionnelle sans couper ni coller. Pratiqué depuis "
+            "le VIe siècle au Japon, il était d'abord réservé aux cérémonies religieuses. "
+            "Les modèles traditionnels incluent la grue (symbole de paix), la grenouille "
+            "et la fleur. L'origami moderne explore la complexité mathématique avec des "
+            "créations hyperréalistes. Cette discipline développe patience, précision et "
+            "compréhension spatiale."
+        ),
+    },
+    {
+        "id": "sc-70",
+        "title": "La Mosaïque",
+        "content": (
+            "La mosaïque assemble de petits fragments colorés (tesselles) de pierre, "
+            "céramique ou verre pour créer des images et motifs. Les Romains et Byzantins "
+            "ont porté cet art à son apogée avec les splendeurs de Ravenne et de "
+            "Constantinople. Chaque tesselle est posée individuellement sur un support "
+            "avec du mortier. Les jeux de lumière sur les tesselles de verre créent des "
+            "effets lumineux uniques. Gaudi a réinventé la mosaïque moderne avec le "
+            "trencadis au Parc Güell."
+        ),
+    },
+    {
+        "id": "sc-71",
+        "title": "Le Théâtre",
+        "content": (
+            "Le théâtre est à la fois un art de la représentation et un lieu de rencontre "
+            "sociale. Né dans l'Antiquité grecque avec les tragédies d'Eschyle et "
+            "Sophocle, il explore les grandes questions humaines. Le Moyen Âge voit "
+            "l'essor des mystères religieux, tandis que la Renaissance célèbre "
+            "Shakespeare et Molière. Le théâtre moderne expérimente avec le réalisme, "
+            "l'absurde et le théâtre de l'opprimé. Il combine texte, jeu d'acteur, décor "
+            "et lumière pour créer une expérience immersive."
+        ),
+    },
+    {
+        "id": "sc-72",
+        "title": "Le Vitrail",
+        "content": (
+            "Le vitrail assemble des morceaux de verre coloré maintenus par des baguettes "
+            "de plomb pour créer des compositions lumineuses. Au Moyen Âge, les "
+            "cathédrales gothiques comme Chartres transforment la lumière divine en "
+            "récits bibliques. Les maîtres verriers maîtrisent la chimie des oxydes "
+            "métalliques pour obtenir des couleurs intenses. Chaque pièce est taillée "
+            "selon le carton préparatoire puis sertie. Le XXe siècle voit Chagall et "
+            "Soulages réinventer cet art millénaire."
+        ),
+    },
+    {
+        "id": "sc-73",
+        "title": "La Gravure",
+        "content": (
+            "La gravure est une technique d'impression où l'artiste incise une matrice "
+            "(bois, métal, pierre) pour créer des estampes multiples. La xylogravie "
+            "(bois) est la plus ancienne, utilisée par Dürer et les estampes japonaises "
+            "ukiyo-e. La taille-douce (métal) comprend l'eau-forte, l'aquatinte et le "
+            "burin, prisées par Rembrandt et Goya. La lithographie, inventée en 1796, "
+            "permet des nuances subtiles exploitées par Toulouse-Lautrec. Chaque tirage "
+            "est numéroté et signé par l'artiste."
+        ),
+    },
+]

src/backend/evaluation/data/full_text/__init__.py

@@ -0,0 +1,4 @@
+"""
+evaluation dataset for full_text capabilities.
+evaluation should be good with embeddings disabled
+"""

src/backend/evaluation/data/full_text/documents.py

@@ -0,0 +1,10 @@
+"""document data for full text evaluation"""
+
+from ..corpus.simple_corpus import documents as simple_corpus_documents
+
+documents = [
+    *simple_corpus_documents,
+    {"id": "ft-1", "title": "L'éléphant", "content": "L'éléphant s'est échappé"},
+    {"id": "ft-2", "title": "Foot", "content": "Le foot est un sport populaire"},
+    {"id": "ft-3", "title": "Il va courir", "content": "Il va courir"},
+]

src/backend/evaluation/data/full_text/queries.py

@@ -0,0 +1,26 @@
+"""Queries and expected document IDs for evaluation in French language."""
+
+queries = [
+    {
+        "q": "elephant",
+        "expected_document_ids": ["sc-23", "ft-1"],
+    },
+    {
+        "q": "courir",
+        "expected_document_ids": ["ft-3"],
+    },
+    {
+        # test "football"  -> "foot"
+        "q": "football",
+        "expected_document_ids": ["ft-2"],
+    },
+    {  # test partial word matching
+        "q": "couri",
+        "expected_document_ids": ["ft-3"],
+    },
+    {
+        # test fuzzy matching with ngrams
+        "q": "courrir",
+        "expected_document_ids": ["ft-3"],
+    },
+]

src/backend/evaluation/data/semantic/__init__.py

@@ -0,0 +1 @@
+"""base evaluation datasets for semantic capabilities."""

src/backend/evaluation/data/semantic/documents.py

@@ -0,0 +1,5 @@
+"""Documents for semantic evaluation."""
+
+from ..corpus.simple_corpus import documents as simple_corpus_documents
+
+documents = simple_corpus_documents

src/backend/evaluation/data/semantic/queries.py

@@ -0,0 +1,32 @@
+"""Queries and expected document IDs for evaluation in French language."""
+
+queries = [
+    {
+        "q": "cours d'histoire de l'antiquité",
+        "expected_document_ids": ["sc-5", "sc-8"],
+    },
+    {
+        "q": "recette salée végétarienne",
+        "expected_document_ids": ["sc-42", "sc-54", "sc-58"],
+    },
+    {
+        "q": "art dramatique",
+        "expected_document_ids": ["sc-71"],
+    },
+    {
+        "q": "art de bouger son corps",
+        "expected_document_ids": ["sc-63", "sc-68"],
+    },
+    {
+        "q": "mammifères aquatiques",
+        "expected_document_ids": ["sc-22", "sc-37"],
+    },
+    {
+        "q": "insectes pollinisateurs",
+        "expected_document_ids": ["sc-29"],
+    },
+    {
+        "q": "prédateur félin",
+        "expected_document_ids": ["sc-21", "sc-30"],
+    },
+]

src/backend/evaluation/data/v0/__init__.py

@@ -0,0 +1 @@
+"""evaluation dataset tests"""

src/backend/evaluation/data/v0/documents.py

@@ -0,0 +1,3 @@
+"""document data"""
+
+documents = [{"id": 1, "title": "document", "content": "a document"}]

src/backend/evaluation/data/v0/queries.py

@@ -0,0 +1,8 @@
+"""Queries and expected document IDs for test evaluation."""
+
+queries = [
+    {
+        "q": "a query",
+        "expected_document_ids": [1],
+    },
+]

src/backend/evaluation/management/commands/evaluate_search_engine.py

@@ -0,0 +1,229 @@
+"""
+Evaluate search engine performance with test documents and queries.
+"""
+
+import importlib
+import logging
+import math
+
+from django.core.management.base import BaseCommand
+
+from core.enums import SearchTypeEnum
+from core.management.commands.create_search_pipeline import (
+    ensure_search_pipeline_exists,
+)
+from core.services.opensearch import (
+    check_hybrid_search_enabled,
+    opensearch_client,
+)
+from core.services.search import (
+    search,
+)
+from core.utils import (
+    bulk_create_documents,
+    delete_index,
+    delete_search_pipeline,
+    get_language_value,
+    prepare_index,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """Evaluate search engine performance"""
+
+    help = __doc__
+    opensearch_client_ = opensearch_client()
+    index_name = "evaluation-index"
+    search_params = {
+        "nb_results": 20,
+        "search_indices": {index_name},
+        "reach": None,
+        "user_sub": "user_sub",
+        "groups": [],
+        "visited": [],
+        "tags": [],
+    }
+    base_data_path = "evaluation/data"
+    documents = []
+    queries = []
+    id_to_title = {}
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            dest="dataset_name",
+            type=str,
+            help="Name of the dataset to use for evaluation",
+        )
+        parser.add_argument(
+            "--min_score",
+            dest="min_score",
+            type=float,
+            default=0.0,
+            help="hits with a score lower than min_score are ignored",
+        )
+        parser.add_argument(
+            "--keep-index",
+            dest="keep_index",
+            type=bool,
+            default=True,
+            help="If True the index is not dropped after evaluation.",
+        )
+        parser.add_argument(
+            "--force-reindex",
+            dest="force_reindex",
+            type=bool,
+            default=False,
+            help=(
+                "If True the index is dropped and recreated from scratch before evaluation."
+            ),
+        )
+
+    def handle(self, *args, **options):
+        """Launch the search engine evaluation."""
+
+        self.init_evaluation(options["dataset_name"], options["force_reindex"])
+        self.stdout.write(
+            f"[INFO] Starting evaluation with {len(self.documents)} "
+            f"documents and {len(self.queries)} queries"
+        )
+
+        evaluations = [
+            self.evaluate_query(query, options["min_score"]) for query in self.queries
+        ]
+
+        avg_metrics = self.calculate_average_metrics(evaluations)
+        self.stdout.write(
+            f"\n{'=' * 60}\n"
+            f"[SUMMARY] Average Performance\n"
+            f"{'=' * 60}\n"
+            f"  Average NDCG: {avg_metrics['avg_ndcg']:.2%}\n"
+            f"  Average Precision: {avg_metrics['avg_precision']:.2%}\n"
+            f"  Average Recall: {avg_metrics['avg_recall']:.2%}\n"
+            f"  Average F1-Score: {avg_metrics['avg_f1_score']:.2%}\n"
+        )
+
+        self.close_evaluation(options["keep_index"])
+        self.stdout.write(self.style.SUCCESS("\n[SUCCESS] Evaluation completed"))
+
+    def init_evaluation(self, dataset_name, force_reindex):
+        """Initialize evaluation by preparing index and mapping."""
+        self.documents = (
+            importlib.import_module(f"evaluation.data.{dataset_name}.documents")
+        ).documents
+        self.queries = (
+            importlib.import_module(f"evaluation.data.{dataset_name}.queries")
+        ).queries
+        self.id_to_title = {
+            document["id"]: document["title"] for document in self.documents
+        }
+        check_hybrid_search_enabled.cache_clear()
+        delete_search_pipeline()
+        ensure_search_pipeline_exists()
+        if not opensearch_client().indices.exists(index=self.index_name):
+            prepare_index(self.index_name, bulk_create_documents(self.documents))
+        elif force_reindex:
+            delete_index(self.index_name)
+            prepare_index(self.index_name, bulk_create_documents(self.documents))
+
+    def evaluate_query(self, query, min_score=0.0):
+        """Evaluate a single query and return metrics."""
+        results = search(
+            q=query["q"],
+            search_type=SearchTypeEnum.HYBRID
+            if check_hybrid_search_enabled()
+            else SearchTypeEnum.FULL_TEXT,
+            **self.search_params,
+        )
+        expected_titles = [
+            self.id_to_title[document_id]
+            for document_id in query["expected_document_ids"]
+        ]
+        retrieved_ordered_titles = [
+            get_language_value(result["_source"], "title")
+            for result in results["hits"]["hits"]
+            if result["_score"] >= min_score
+        ]
+
+        metrics = self.calculate_metrics(expected_titles, retrieved_ordered_titles)
+
+        self.stdout.write(
+            f"  [QUERY EVALUATION]\n"
+            f"  q: {query['q']}\n"
+            f"  expect: {list(expected_titles)}\n"
+            f"  result: {list(retrieved_ordered_titles)}\n"
+            f"  NDCG: {metrics['ndcg']:.2%} \n"
+            f"  PRECISION: {metrics['precision']:.2%} \n"
+            f"  RECALL: {metrics['recall']:.2%} \n"
+            f"  F1-SCORE: {metrics['f1_score']:.2%} \n"
+        )
+        return {
+            "q": query["q"],
+            "expected_titles": expected_titles,
+            "retrieved_titles": retrieved_ordered_titles,
+            "metrics": metrics,
+        }
+
+    def calculate_metrics(self, expected_titles, retrieved_ordered_titles):
+        """Calculate precision, recall, F1-score, DCG and NDCG."""
+
+        dcg = self.calculate_dcg(expected_titles, retrieved_ordered_titles)
+        idcg = self.calculate_dcg(expected_titles, expected_titles)
+        ndcg = dcg / idcg if idcg > 0 else 0
+        nb_true_positives = len(set(expected_titles) & set(retrieved_ordered_titles))
+        precision = (
+            nb_true_positives / len(retrieved_ordered_titles)
+            if retrieved_ordered_titles
+            else 0
+        )
+        recall = nb_true_positives / len(expected_titles) if expected_titles else 0
+        f1_score = (
+            2 * (precision * recall) / (precision + recall)
+            if (precision + recall) > 0
+            else 0
+        )
+
+        return {
+            "ndcg": ndcg,
+            "precision": precision,
+            "recall": recall,
+            "f1_score": f1_score,
+            "true_positives": nb_true_positives,
+        }
+
+    def calculate_dcg(self, expected_titles, retrieved_ordered_titles):
+        """Calculate Discounted Cumulative Gain."""
+        return sum(
+            (1 if title in expected_titles else 0) / math.log2(rank + 2)
+            for rank, title in enumerate(retrieved_ordered_titles)
+        ) / len(expected_titles)
+
+    def calculate_average_metrics(self, evaluations):
+        """Calculate average metrics across all queries."""
+        if not evaluations:
+            return {
+                "avg_ndcg": 0,
+                "avg_precision": 0,
+                "avg_recall": 0,
+                "avg_f1_score": 0,
+            }
+
+        total_ndcg = sum(r["metrics"]["ndcg"] for r in evaluations)
+        total_precision = sum(r["metrics"]["precision"] for r in evaluations)
+        total_recall = sum(r["metrics"]["recall"] for r in evaluations)
+        total_f1 = sum(r["metrics"]["f1_score"] for r in evaluations)
+        nb_evaluations = len(evaluations)
+
+        return {
+            "avg_ndcg": total_ndcg / nb_evaluations,
+            "avg_precision": total_precision / nb_evaluations,
+            "avg_recall": total_recall / nb_evaluations,
+            "avg_f1_score": total_f1 / nb_evaluations,
+        }
+
+    def close_evaluation(self, keep_index):
+        """Delete the evaluation index."""
+        delete_search_pipeline()
+        if not keep_index:
+            delete_index(self.index_name)

src/backend/evaluation/tests/test_evaluate_search_engine.py

@@ -0,0 +1,134 @@
+"""
+Test suite for evaluate_search_engine management command
+"""
+
+import io
+import logging
+from unittest.mock import patch
+
+from django.core.management import call_command
+
+import pytest
+import responses
+
+from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
+from core.utils import delete_index, delete_search_pipeline
+
+logger = logging.getLogger(__name__)
+
+pytestmark = pytest.mark.django_db
+
+
+INDEX_NAME = "evaluation-index"
+
+
+@pytest.fixture(autouse=True)
+def clear_caches_and_cleanup():
+    """Clear caches and cleanup before and after each test"""
+    clear()
+    yield
+    clear()
+
+
+@pytest.fixture(autouse=True)
+def disable_hybrid_search(settings):
+    """Disable hybrid search for all tests to prevent API calls"""
+    settings.HYBRID_SEARCH_ENABLED = False
+
+
+def clear():
+    """Clear caches and delete index and pipeline"""
+    check_hybrid_search_enabled.cache_clear()
+    delete_search_pipeline()
+    delete_index(INDEX_NAME)
+
+
+def assert_output_successful(output):
+    """Assert that the output indicates a successful evaluation"""
+    assert "[INFO] Starting evaluation with 1 documents and 1 queries" in output
+    assert "[QUERY EVALUATION]" in output
+    assert "q: a query" in output
+    assert "[SUMMARY] Average Performance" in output
+    assert "Average NDCG:" in output
+    assert "Average Precision:" in output
+    assert "Average Recall:" in output
+    assert "Average F1-Score:" in output
+    assert "[SUCCESS] Evaluation completed" in output
+
+
+@responses.activate
+def test_evaluate_search_engine_command_v0():
+    """Test running the evaluate_search_engine command with v0 dataset"""
+    out = io.StringIO()
+    call_command(
+        "evaluate_search_engine",
+        "v0",
+        stdout=out,
+    )
+
+    assert_output_successful(out.getvalue())
+
+    # Index should still exist because keep-index is True by default
+    assert opensearch_client().indices.exists(index="evaluation-index")
+
+
+@responses.activate
+def test_evaluate_search_engine_command_without_keep_index():
+    """Test that keep-index option False erases index"""
+    out = io.StringIO()
+    call_command(
+        "evaluate_search_engine",
+        "v0",
+        keep_index=False,
+        stdout=out,
+    )
+
+    assert_output_successful(out.getvalue())
+
+    # Index should not exist
+    assert not opensearch_client().indices.exists(index="evaluation-index")
+
+
+@patch("evaluation.management.commands.evaluate_search_engine.delete_index")
+@responses.activate
+def test_evaluate_search_engine_command_force_reindex(mock_delete_index):
+    """Test that force-reindex must delete and recreates the index"""
+    out = io.StringIO()
+    # run once to create the index
+    call_command(
+        "evaluate_search_engine",
+        "v0",
+        stdout=out,
+    )
+
+    mock_delete_index.clear()
+    # Run again with force-reindex
+    call_command(
+        "evaluate_search_engine",
+        "v0",
+        force_reindex=True,
+        stdout=out,
+    )
+
+    # Verify delete_index was called once with the correct index name
+    mock_delete_index.assert_called_once_with("evaluation-index")
+
+
+@responses.activate
+def test_evaluate_search_engine_min_score_filter():
+    """Test that min_score filters out low-scoring results"""
+
+    out = io.StringIO()
+    super_high_score = 1000.0
+    call_command(
+        "evaluate_search_engine",
+        "v0",
+        min_score=super_high_score,
+        stdout=out,
+    )
+
+    # Assert all scores are null proving all results were filtered out
+    assert (
+        "NDCG: 0.00% \n  PRECISION: 0.00% \n  RECALL: 0.00% \n  F1-SCORE: 0.00%"
+        in out.getvalue()
+    )

src/backend/find/settings.py

@@ -19,6 +19,7 @@ from django.utils.translation import gettext_lazy as _
 import sentry_sdk
 from configurations import Configuration, values
 from sentry_sdk.integrations.django import DjangoIntegration
+from sentry_sdk.integrations.logging import ignore_logger
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -123,16 +124,24 @@ class Base(Configuration):
     # https://docs.djangoproject.com/en/3.1/topics/i18n/
 
     # Languages
-    LANGUAGE_CODE = values.Value("en-us")
-
-    # Careful! Languages should be ordered by priority, as this tuple is used to get
-    # fallback/default languages throughout the app.
     LANGUAGES = values.SingleNestedTupleValue(
         (
-            ("en-us", _("English")),
-            ("fr-fr", _("French")),
+            ("fr", _("French")),
+            ("en", _("English")),
+            ("de", _("German")),
+            ("nl", _("Dutch")),
+            ("und", None),
         )
     )
+    SUPPORTED_LANGUAGE_CODES = tuple(
+        language_code for language_code, _ in LANGUAGES.value
+    )
+    LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD = values.FloatValue(
+        default=0.75,
+        environ_name="LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD",
+        environ_prefix=None,
+    )
+    UNDETERMINED_LANGUAGE_CODE = "und"
 
     LOCALE_PATHS = (os.path.join(BASE_DIR, "locale"),)
 
@@ -187,6 +196,7 @@ class Base(Configuration):
         # find
         "core",
         "demo",
+        "evaluation",
         # Third party apps
         "corsheaders",
         "dockerflow.django",
@@ -265,6 +275,14 @@ class Base(Configuration):
 
     AUTH_USER_MODEL = "core.User"
 
+    # Trigrams search settings
+    TRIGRAMS_BOOST = values.Value(
+        default=0.25, environ_name="TRIGRAMS_BOOST", environ_prefix=None
+    )
+    TRIGRAMS_MINIMUM_SHOULD_MATCH = values.Value(
+        default="75%", environ_name="TRIGRAMS_MINIMUM_SHOULD_MATCH", environ_prefix=None
+    )
+
     # Hybrid Search settings
     HYBRID_SEARCH_ENABLED = values.BooleanValue(
         default=False, environ_name="HYBRID_SEARCH_ENABLED", environ_prefix=None
@@ -273,6 +291,13 @@ class Base(Configuration):
     HYBRID_SEARCH_WEIGHTS = values.ListValue(
         default=[0.3, 0.7], environ_name="HYBRID_SEARCH_WEIGHTS", environ_prefix=None
     )
+    # Multi-embedding: chunk documents and embed each chunk separately
+    CHUNK_SIZE = values.IntegerValue(
+        default=512, environ_name="CHUNK_SIZE", environ_prefix=None
+    )
+    CHUNK_OVERLAP = values.IntegerValue(
+        default=50, environ_name="CHUNK_OVERLAP", environ_prefix=None
+    )
     EMBEDDING_API_PATH = values.Value(
         # embedding is the vector representation of a document used for semantic search
         default="None",
@@ -282,7 +307,7 @@ class Base(Configuration):
     EMBEDDING_API_KEY = values.Value(
         default=None, environ_name="EMBEDDING_API_KEY", environ_prefix=None
     )
-    EMBEDDING_REQUEST_TIMEOUT = values.Value(
+    EMBEDDING_REQUEST_TIMEOUT = values.IntegerValue(
         default=10, environ_name="EMBEDDING_REQUEST_TIMEOUT", environ_prefix=None
     )
     EMBEDDING_API_MODEL_NAME = values.Value(
@@ -293,6 +318,19 @@ class Base(Configuration):
     EMBEDDING_DIMENSION = values.IntegerValue(
         default=1024, environ_name="EMBEDDING_DIMENSION", environ_prefix=None
     )
+    # rescore
+    RESCORE_UPDATED_AT_WEIGHT = values.FloatValue(
+        default=0.2, environ_name="RESCORE_UPDATED_AT_WEIGHT", environ_prefix=None
+    )
+    RESCORE_UPDATED_AT_OFFSET = values.Value(
+        default="2d", environ_name="RESCORE_UPDATED_AT_OFFSET", environ_prefix=None
+    )
+    RESCORE_UPDATED_AT_SCALE = values.Value(
+        default="6d", environ_name="RESCORE_UPDATED_AT_SCALE", environ_prefix=None
+    )
+    RESCORE_UPDATED_AT_DECAY = values.IntegerValue(
+        default=0.5, environ_name="RESCORE_UPDATED_AT_SCALE", environ_prefix=None
+    )
 
     # CORS
     CORS_ALLOW_CREDENTIALS = True
@@ -301,7 +339,7 @@ class Base(Configuration):
     CORS_ALLOWED_ORIGIN_REGEXES = values.ListValue([])
 
     # Sentry
-    SENTRY_DSN = values.Value(None, environ_name="SENTRY_DSN")
+    SENTRY_DSN = values.Value(None, environ_name="SENTRY_DSN", environ_prefix=None)
 
     # Celery
     CELERY_BROKER_URL = values.Value("redis://redis:6379/0")
@@ -507,8 +545,11 @@ class Base(Configuration):
                 release=get_release(),
                 integrations=[DjangoIntegration()],
             )
-            with sentry_sdk.configure_scope() as scope:
-                scope.set_extra("application", "backend")
+            scope = sentry_sdk.get_current_scope()
+            scope.set_extra("application", "backend")
+
+            # Ignore the logs added by the DockerflowMiddleware
+            ignore_logger("request.summary")
 
 
 class Build(Base):
@@ -586,6 +627,9 @@ class Production(Base):
     """
 
     # Security
+    # Add allowed host from environment variables.
+    # The machine hostname is added by default,
+    # it makes the application pingable by a load balancer on the same machine by example
     ALLOWED_HOSTS = [
         *values.ListValue([], environ_name="ALLOWED_HOSTS"),
         gethostbyname(gethostname()),
@@ -605,6 +649,14 @@ class Production(Base):
     # In other cases, you should comment the following line to avoid security issues.
     # SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
     SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
+    SECURE_HSTS_SECONDS = 60
+    SECURE_HSTS_PRELOAD = True
+    SECURE_HSTS_INCLUDE_SUBDOMAINS = True
+    SECURE_SSL_REDIRECT = True
+    SECURE_REDIRECT_EXEMPT = [
+        "^__lbheartbeat__",
+        "^__heartbeat__",
+    ]
 
     # Modern browsers require to have the `secure` attribute on cookies with `Samesite=none`
     CSRF_COOKIE_SECURE = True
@@ -621,6 +673,11 @@ class Production(Base):
                 environ_name="REDIS_URL",
                 environ_prefix=None,
             ),
+            "TIMEOUT": values.IntegerValue(
+                30,  # timeout in seconds
+                environ_name="CACHES_DEFAULT_TIMEOUT",
+                environ_prefix=None,
+            ),
             "OPTIONS": {
                 "CLIENT_CLASS": "django_redis.client.DefaultClient",
             },

src/backend/pyproject.toml

@@ -17,35 +17,37 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Natural Language :: English",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.12",
 ]
 description = "An application to print markdown to pdf from a set of managed templates."
 keywords = ["Django", "Contacts", "Templates", "RBAC"]
 license = { file = "LICENSE" }
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = "~=3.12.0"
 dependencies = [
-    "celery[redis]==5.5.3",
+    "celery[redis]==5.6.2",
     "django-configurations==2.5.1",
-    "django-cors-headers==4.7.0",
-    "redis==5.2.1",
+    "django-cors-headers==4.9.0",
+    "redis==5.3.1",
     "django-redis==6.0.0",
-    "django==5.2.6",
-    "django-lasuite[all]==0.0.14",
-    "djangorestframework==3.16.0",
-    "drf_spectacular==0.28.0",
+    "django==5.2.12",
+    "django-lasuite[all]==0.0.22",
+    "djangorestframework==3.16.1",
+    "drf_spectacular==0.29.0",
     "dockerflow==2024.4.2",
-    "factory_boy==3.3.1",
+    "factory_boy==3.3.3",
     "gunicorn==23.0.0",
-    "mozilla-django-oidc==4.0.1",
-    "psycopg[binary]==3.2.9",
-    "pydantic==2.10.5",
+    "py3langid==0.3.0",
+    "langchain-text-splitters==1.1.0",
+    "mozilla-django-oidc==5.0.2",
+    "psycopg[binary]==3.3.2",
+    "pydantic==2.12.5",
     "pyjwt==2.10.1",
-    "requests==2.32.4",
-    "sentry-sdk==2.32.0",
-    "url-normalize==1.4.3",
-    "opensearch-py==2.8.0",
-    "whitenoise==6.8.2",
+    "requests==2.32.5",
+    "sentry-sdk==2.48.0",
+    "url-normalize==2.2.1",
+    "opensearch-py==3.1.0",
+    "whitenoise==6.11.0",
 ]
 
 [project.urls]
@@ -57,21 +59,21 @@ dependencies = [
 [project.optional-dependencies]
 dev = [
     "django-extensions==4.1",
-    "drf-spectacular-sidecar==2025.7.1",
-    "faker==33.3.0",
+    "drf-spectacular-sidecar==2026.1.1",
+    "faker==40.1.0",
     "ipdb==0.13.13",
-    "ipython==8.31.0",
-    "pyfakefs==5.9.1",
-    "pylint-django==2.6.1",
-    "pylint==3.3.7",
-    "pytest-cov==6.2.1",
+    "ipython==9.8.0",
+    "pyfakefs==6.0.0",
+    "pylint-django==2.7.0",
+    "pylint==4.0.4",
+    "pytest-cov==7.0.0",
     "pytest-django==4.11.1",
-    "pytest==8.4.1",
+    "pytest==9.0.2",
     "pytest-icdiff==0.9",
     "pytest-xdist==3.8.0",
-    "responses==0.25.7",
-    "ruff==0.12.2",
-    "types-requests==2.32.4.20250611",
+    "responses==0.25.8",
+    "ruff==0.14.10",
+    "types-requests==2.32.4.20250913",
 ]
 
 [tool.setuptools]

src/backend/setup.py

@@ -1,7 +0,0 @@
-#!/usr/bin/env python
-"""Setup file for the find module. All configuration stands in the setup.cfg file."""
-# coding: utf-8
-
-from setuptools import setup  # pylint: disable=import-error
-
-setup()

src/helm/find/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v2
 type: application
 name: find
-version: 0.0.2
+version: 0.0.3

src/helm/find/README.md

@@ -43,6 +43,7 @@
 | `backend.shareProcessNamespace`                       | Enable share process namespace between containers                                  | `false`                                         |
 | `backend.sidecars`                                    | Add sidecars containers to backend deployment                                      | `[]`                                            |
 | `backend.migrateJobAnnotations`                       | Annotations for the migrate job                                                    | `{}`                                            |
+| `backend.podSecurityContext`                          | Configure backend Pod security context                                             | `nil`                                           |
 | `backend.securityContext`                             | Configure backend Pod security context                                             | `nil`                                           |
 | `backend.envVars`                                     | Configure backend container environment variables                                  | `undefined`                                     |
 | `backend.envVars.BY_VALUE`                            | Example environment variable by setting value directly                             |                                                 |
@@ -58,15 +59,15 @@
 | `backend.migrate.command`                             | backend migrate command                                                            | `["python","manage.py","migrate","--no-input"]` |
 | `backend.migrate.restartPolicy`                       | backend migrate job restart policy                                                 | `Never`                                         |
 | `backend.probes.liveness.path`                        | Configure path for backend HTTP liveness probe                                     | `/__heartbeat__`                                |
-| `backend.probes.liveness.targetPort`                  | Configure port for backend HTTP liveness probe                                     | `undefined`                                     |
+| `backend.probes.liveness.targetPort`                  | Configure port for backend HTTP liveness probe                                     | `nil`                                           |
 | `backend.probes.liveness.initialDelaySeconds`         | Configure initial delay for backend liveness probe                                 | `10`                                            |
 | `backend.probes.liveness.initialDelaySeconds`         | Configure timeout for backend liveness probe                                       | `10`                                            |
-| `backend.probes.startup.path`                         | Configure path for backend HTTP startup probe                                      | `undefined`                                     |
-| `backend.probes.startup.targetPort`                   | Configure port for backend HTTP startup probe                                      | `undefined`                                     |
-| `backend.probes.startup.initialDelaySeconds`          | Configure initial delay for backend startup probe                                  | `undefined`                                     |
-| `backend.probes.startup.initialDelaySeconds`          | Configure timeout for backend startup probe                                        | `undefined`                                     |
+| `backend.probes.startup.path`                         | Configure path for backend HTTP startup probe                                      | `nil`                                           |
+| `backend.probes.startup.targetPort`                   | Configure port for backend HTTP startup probe                                      | `nil`                                           |
+| `backend.probes.startup.initialDelaySeconds`          | Configure initial delay for backend startup probe                                  | `nil`                                           |
+| `backend.probes.startup.initialDelaySeconds`          | Configure timeout for backend startup probe                                        | `nil`                                           |
 | `backend.probes.readiness.path`                       | Configure path for backend HTTP readiness probe                                    | `/__lbheartbeat__`                              |
-| `backend.probes.readiness.targetPort`                 | Configure port for backend HTTP readiness probe                                    | `undefined`                                     |
+| `backend.probes.readiness.targetPort`                 | Configure port for backend HTTP readiness probe                                    | `nil`                                           |
 | `backend.probes.readiness.initialDelaySeconds`        | Configure initial delay for backend readiness probe                                | `10`                                            |
 | `backend.probes.readiness.initialDelaySeconds`        | Configure timeout for backend readiness probe                                      | `10`                                            |
 | `backend.resources`                                   | Resource requirements for the backend container                                    | `{}`                                            |

src/helm/find/generate-readme.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 docker image ls | grep readme-generator-for-helm
 if [ "$?" -ne "0" ]; then

src/helm/find/templates/backend_deployment.yaml

@@ -90,6 +90,10 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.backend.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.backend.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

src/helm/find/templates/backend_job.yaml

@@ -75,6 +75,10 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.backend.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.backend.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

src/helm/find/templates/backend_job_createsuperuser.yaml

@@ -76,6 +76,10 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.backend.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.backend.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

src/helm/find/values.yaml

@@ -92,6 +92,9 @@ backend:
   ## @param backend.migrateJobAnnotations Annotations for the migrate job
   migrateJobAnnotations: {}
 
+  ## @param backend.podSecurityContext Configure backend Pod security context
+  podSecurityContext: null
+
   ## @param backend.securityContext Configure backend Pod security context
   securityContext: null