🔧(backend) setup Find app dockers to work with Docs

Populate a service configuration "docs" for development.
Move OIDC endpoints to the "impress" realm to allow introspection
of Docs tokens.
Upgrade version of dependencies (fix some security issues).

Signed-off-by: Fabre Florian <ffabre@hybird.org>

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,5 +7,5 @@ Description...
 
 Description...
 
-- [ ] item 1...
-- [ ] item 2...
+- [] item 1...
+- [] item 2...

.github/workflows/deploy.yml

@@ -0,0 +1,52 @@
+name: Deploy
+
+on:
+  push:
+    tags:
+      - 'preprod'
+      - 'production'
+
+
+jobs:
+  notify-argocd:
+    runs-on: ubuntu-latest
+    steps:
+      -
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.APP_ID }}
+          private-key: ${{ secrets.PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: "drive,secrets"
+      -
+        name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          submodules: recursive
+          token: ${{ steps.app-token.outputs.token }}
+      -
+        name: Load sops secrets
+        uses: rouja/actions-sops@main
+        with:
+          secret-file: secrets/numerique-gouv/drive/secrets.enc.env
+          age-key: ${{ secrets.SOPS_PRIVATE }}
+      -
+        name: Call argocd github webhook
+        run: |
+          data='{"ref": "'$GITHUB_REF'","repository": {"html_url":"'$GITHUB_SERVER_URL'/'$GITHUB_REPOSITORY'"}}'
+          sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}')
+          curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_WEBHOOK_URL
+          sig=$(echo -n ${data} | openssl dgst -sha1 -hmac ''${ARGOCD_PRODUCTION_WEBHOOK_SECRET}'' | awk '{print "X-Hub-Signature: sha1="$2}')
+          curl -X POST -H 'X-GitHub-Event:push' -H "Content-Type: application/json" -H "${sig}" --data "${data}" $ARGOCD_PRODUCTION_WEBHOOK_URL
+
+  start-test-on-preprod:
+    needs:
+      - notify-argocd
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.ref, 'refs/tags/preprod')
+    steps:
+      -
+        name: Debug
+        run: |
+          echo "Start test when preprod is ready"

.github/workflows/docker-hub.yml

@@ -20,13 +20,7 @@ jobs:
     steps:
       -
         name: Checkout repository
-        uses: actions/checkout@v6
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: actions/checkout@v4
       -
         name: Docker meta
         id: meta
@@ -50,7 +44,6 @@ jobs:
         with:
           context: .
           target: backend-production
-          platforms: linux/amd64,linux/arm64
           build-args: DOCKER_USER=${{ env.DOCKER_USER }}:-1000
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}

.github/workflows/find.yml

@@ -14,7 +14,7 @@ jobs:
     if: github.event_name == 'pull_request' # Makes sense only for pull requests
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - name: show
@@ -37,7 +37,7 @@ jobs:
       github.event_name == 'pull_request'
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v3
         with:
           fetch-depth: 50
       - name: Check that the CHANGELOG has been modified in the current branch
@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v2
       - name: Check CHANGELOG max line length
         run: |
           max_line_length=$(cat CHANGELOG.md | grep -Ev "^\[.*\]: https://github.com" | wc -L)
@@ -63,22 +63,19 @@ jobs:
         working-directory: src/backend
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
-      - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/checkout@v2
+      - name: Install Python
+        uses: actions/setup-python@v3
         with:
-          python-version-file: "src/backend/pyproject.toml"
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
-      - name: Install the project
-        run: uv sync --locked --all-extras
-
+          python-version: "3.12"
+      - name: Install development dependencies
+        run: pip install --user .[dev]
       - name: Check code formatting with ruff
-        run: uv run ruff format . --diff
+        run: ~/.local/bin/ruff format . --diff
       - name: Lint code with ruff
-        run: uv run ruff check .
+        run: ~/.local/bin/ruff check .
       - name: Lint code with pylint
-        run: uv run pylint .
+        run: ~/.local/bin/pylint .
 
   test-back:
     runs-on: ubuntu-latest
@@ -108,11 +105,6 @@ jobs:
           OPENSEARCH_INITIAL_ADMIN_PASSWORD: find.PASS123
         ports:
           - 9200:9200
-        options: >-
-          --health-cmd "curl -s http://localhost:9200 > /dev/null || exit 1"
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 10
 
     env:
       DJANGO_CONFIGURATION: Test
@@ -131,35 +123,20 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: Create writable /data
         run: |
           sudo mkdir -p /data/media && \
           sudo mkdir -p /data/static
 
-      - name: Set up Python
-        uses: actions/setup-python@v6
+      - name: Install Python
+        uses: actions/setup-python@v3
         with:
-          python-version-file: "src/backend/pyproject.toml"
-      - name: Install uv
-        uses: astral-sh/setup-uv@v6
+          python-version: "3.12"
 
-      - name: Wait for OpenSearch to be ready
-        run: |
-          for i in {1..60}; do
-            if curl -s http://localhost:9200 > /dev/null; then
-              echo "OpenSearch is ready"
-              exit 0
-            fi
-            echo "Waiting for OpenSearch... attempt $i"
-            sleep 1
-          done
-          echo "OpenSearch failed to start"
-          exit 1
-
-      - name: Install the dependencies
-        run: uv sync --locked --all-extras
+      - name: Install development dependencies
+        run: pip install --user .[dev]
 
       - name: Run tests
-        run: uv run pytest
+        run: ~/.local/bin/pytest

.github/workflows/release-helm-chart.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 

.gitignore

@@ -69,7 +69,6 @@ src/frontend/tsclient
 .pylint.d
 .pytest_cache
 db.sqlite3
-*history.sqlite    
 .mypy_cache
 
 # Site media
@@ -80,7 +79,3 @@ db.sqlite3
 .vscode/
 *.iml
 .devcontainer
-.vscode-server
-
-#pdb
-*.pdbhistory

CHANGELOG.md

@@ -10,35 +10,12 @@ and this project adheres to
 
 ## Added
 
-- 👷(docker) add arm64 platform support for image builds
-- ✨(backend) add semantic search
-- ✨(backend) add multi-embedding and chunking
-- ✨(backend) add analyzers to full-text search
-- ✨(backend) handle french, english, german and dutch 
-- ✨(backend) add evaluation command
 - backend application
 - helm chart
+- 🐛(backend) fix missing index creation in 'index/' view
 - ✨(backend) allow indexation of documents with either empty content or title.
 - ✨(api) new fulltext 'search/' view with OIDC resource server authentication
 - ✨(backend) limit access to documents : public & authenticated with a
+              linkreach & owned ones
+- ✨(backend) limit search to the calling app (audience) and a configured
               list of services
-- 🔧(compose) rename docker network 'lasuite-net' as 'lasuite-network'
-- ✨(backend) add demo service for Drive.
-- ✨(backend) add OPENSEARCH_INDEX_PREFIX setting to prevent naming overlaping
-              issues if the opensearch database is shared between apps.
-- ✨(backend) add tags
-- ✨(backend) adapt to conversation RAG
-- ✨(backend) add deletion endpoint
-- ✨(backend) add path filter
-- ✨(backend) add search_type param
-
-## Changed
-
-- 🏗️(backend) switch Python dependency management to uv
-- ✨(backend) allow deletion by tags
-- ♻️(backend) improve the evaluation command
-
-## Fixed
-
-- 🐛(backend) fix missing index creation in 'index/' view
-- 🐛(backend) fix parallel test execution issues

Dockerfile

@@ -3,6 +3,9 @@
 # ---- base image to inherit from ----
 FROM python:3.12-slim-bookworm AS base
 
+# Upgrade pip to its latest release to speed up dependencies installation
+RUN python -m pip install --upgrade pip
+
 # Upgrade system packages to install security updates
 RUN apt-get update && \
   apt-get -y upgrade && \
@@ -11,28 +14,13 @@ RUN apt-get update && \
 # ---- Back-end builder image ----
 FROM base AS back-builder
 
-ENV UV_COMPILE_BYTECODE=1
-ENV UV_LINK_MODE=copy
+WORKDIR /builder
 
-# Disable Python downloads, because we want to use the system interpreter
-# across both images. If using a managed Python version, it needs to be
-# copied from the build image into the final image;
-ENV UV_PYTHON_DOWNLOADS=0
+# Copy required python dependencies
+COPY ./src/backend /builder
 
-# install uv
-COPY --from=ghcr.io/astral-sh/uv:0.9.10 /uv /uvx /bin/
-
-WORKDIR /app
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=src/backend/uv.lock,target=uv.lock \
-    --mount=type=bind,source=src/backend/pyproject.toml,target=pyproject.toml \
-    uv sync --locked --no-install-project --no-dev
-
-COPY ./src/backend /app
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv sync --locked --no-dev
+RUN mkdir /install && \
+  pip install --prefix=/install .
 
 # ---- static link collector ----
 FROM base AS link-collector
@@ -45,10 +33,11 @@ RUN apt-get update && \
       rdfind && \
     rm -rf /var/lib/apt/lists/*
 
-# Copy the application from the builder
-COPY --from=back-builder /app /app
+# Copy installed python dependencies
+COPY --from=back-builder /install /usr/local
 
-ENV PATH="/app/.venv/bin:$PATH"
+# Copy find application (see .dockerignore)
+COPY ./src/backend /app/
 
 WORKDIR /app
 
@@ -87,13 +76,14 @@ COPY ./docker/files/usr/local/bin/entrypoint /usr/local/bin/entrypoint
 # docker user (see entrypoint).
 RUN chmod g=u /etc/passwd
 
-# Copy the prepared application (see .dockerignore)
-COPY --from=back-builder /app /app
+# Copy installed python dependencies
+COPY --from=back-builder /install /usr/local
+
+# Copy find application (see .dockerignore)
+COPY ./src/backend /app/
 
 WORKDIR /app
 
-ENV PATH="/app/.venv/bin:$PATH"
-
 # We wrap commands run in this container by the following entrypoint that
 # creates a user on-the-fly with the container user ID (see USER) and root group
 # ID.
@@ -110,9 +100,10 @@ RUN apt-get update && \
     apt-get install -y postgresql-client && \
     rm -rf /var/lib/apt/lists/*
 
-# Install development dependencies
-RUN --mount=from=ghcr.io/astral-sh/uv:0.9.10,source=/uv,target=/bin/uv \
-    uv sync --locked --all-extras
+# Uninstall find and re-install it in editable mode along with development
+# dependencies
+RUN pip uninstall -y find
+RUN pip install -e .[dev]
 
 # Restore the un-privileged user running the application
 ARG DOCKER_USER

Makefile

@@ -173,7 +173,7 @@ superuser: ## Create an admin superuser with password "admin"
 .PHONY: superuser
 
 back-i18n-compile: ## compile the gettext files
-	@$(MANAGE) compilemessages --ignore=".venv/**/*"
+	@$(MANAGE) compilemessages --ignore="venv/**/*"
 .PHONY: back-i18n-compile
 
 back-i18n-generate: ## create the .pot files used for i18n

UPGRADE.md

@@ -0,0 +1,17 @@
+# Upgrade
+
+All instructions to upgrade this project from one release to the next will be
+documented in this file. Upgrades must be run sequentially, meaning you should
+not skip minor/major releases while upgrading (fix releases can be skipped).
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+For most upgrades, you just need to run the django migrations with
+the following command inside your docker container:
+
+`python manage.py migrate`
+
+(Note : in your development environment, you can `make migrate`.)
+
+## [Unreleased]

compose.yml

@@ -64,13 +64,12 @@ services:
       - "8081:8000"
     networks:
       default: {}
-      lasuite:
+      lasuite-net:
         aliases:
           - find
     volumes:
       - ./src/backend:/app
       - ./data/static:/data/static
-      - /app/.venv
     depends_on:
       postgresql:
         condition: service_started
@@ -91,7 +90,6 @@ services:
     volumes:
       - ./src/backend:/app
       - ./data/static:/data/static
-      - /app/.venv
     depends_on:
       - app
 
@@ -107,7 +105,15 @@ services:
     user: "${DOCKER_USER:-1000}"
     working_dir: /app
 
+  node:
+    image: node:18
+    user: "${DOCKER_USER:-1000}"
+    environment:
+      HOME: /tmp
+    volumes:
+      - ".:/app"
+
 networks:
-  lasuite:
-    name: lasuite-network
+  lasuite-net:
+    name: lasuite-net
     driver: bridge

docs/architecture.md

@@ -1,13 +0,0 @@
-## Architecture
-
-### Global system architecture
-
-```mermaid
-flowchart TD
-    Docs -- REST API --> Back("Backend (Django)")
-    Back --> DB("Database (PostgreSQL)")
-    Back -- REST API --> Opensearch
-    Back <--> Celery --> DB
-    User -- HTTP --> Dashboard --> Opensearch
-    Back -- REST API --> Embedding Endpoint
-```

docs/env.md

@@ -1,116 +0,0 @@
-# Find variables
-
-Here we describe all environment variables that can be set for the find application.
-
-## find-backend container
-
-These are the environment variables you can set for the `find-backend` container.
-
-| Option                                          | Description                                                                                                                 | default                                                                 |
-|-------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------|
-| API_USERS_LIST_LIMIT                            | Limit on API users                                                                                                          | 5                                                                       |
-| API_USERS_LIST_THROTTLE_RATE_BURST              | Throttle rate for api on burst                                                                                              | 30/minute                                                               |
-| API_USERS_LIST_THROTTLE_RATE_SUSTAINED          | Throttle rate for api                                                                                                       | 180/hour                                                                |
-| CACHES_DEFAULT_TIMEOUT                          | Cache default timeout                                                                                                       | 30                                                                      |
-| CACHES_KEY_PREFIX                               | The prefix used to every cache keys.                                                                                        | docs                                                                    |
-| CHUNK_SIZE                                      | approximate number of characters of document content chunks                                                                 | 512                                                                     |
-| CHUNK_OVERLAP                                   | approximate number of characters of document content overlapping                                                            | 50                                                                      |
-| DB_ENGINE                                       | Engine to use for database connections                                                                                      | django.db.backends.postgresql_psycopg2                                  |
-| DB_HOST                                         | Host of the database                                                                                                        | localhost                                                               |
-| DB_NAME                                         | Name of the database                                                                                                        | impress                                                                 |
-| DB_PASSWORD                                     | Password to authenticate with                                                                                               | pass                                                                    |
-| DB_PORT                                         | Port of the database                                                                                                        | 5432                                                                    |
-| DB_USER                                         | User to authenticate with                                                                                                   | dinum                                                                   |
-| DJANGO_ALLOWED_HOSTS                            | Allowed hosts                                                                                                               | []                                                                      |
-| DJANGO_CELERY_BROKER_TRANSPORT_OPTIONS          | Celery broker transport options                                                                                             | {}                                                                      |
-| DJANGO_CELERY_BROKER_URL                        | Celery broker url                                                                                                           | redis://redis:6379/0                                                    |
-| DJANGO_CORS_ALLOW_ALL_ORIGINS                   | Allow all CORS origins                                                                                                      | false                                                                   |
-| DJANGO_CORS_ALLOWED_ORIGIN_REGEXES              | List of origins allowed for CORS using regulair expressions                                                                 | []                                                                      |
-| DJANGO_CORS_ALLOWED_ORIGINS                     | List of origins allowed for CORS                                                                                            | []                                                                      |
-| DJANGO_CSRF_TRUSTED_ORIGINS                     | CSRF trusted origins                                                                                                        | []                                                                      |
-| DJANGO_EMAIL_BACKEND                            | Email backend library                                                                                                       | django.core.mail.backends.smtp.EmailBackend                             |
-| DJANGO_EMAIL_BRAND_NAME                         | Brand name for email                                                                                                        |                                                                         |
-| DJANGO_EMAIL_FROM                               | Email address used as sender                                                                                                | from@example.com                                                        |
-| DJANGO_EMAIL_HOST                               | Hostname of email                                                                                                           |                                                                         |
-| DJANGO_EMAIL_HOST_PASSWORD                      | Password to authenticate with on the email host                                                                             |                                                                         |
-| DJANGO_EMAIL_HOST_USER                          | User to authenticate with on the email host                                                                                 |                                                                         |
-| DJANGO_EMAIL_LOGO_IMG                           | Logo for the email                                                                                                          |                                                                         |
-| DJANGO_EMAIL_PORT                               | Port used to connect to email host                                                                                          |                                                                         |
-| DJANGO_EMAIL_USE_SSL                            | Use ssl for email host connection                                                                                           | false                                                                   |
-| DJANGO_EMAIL_USE_TLS                            | Use tls for email host connection                                                                                           | false                                                                   |
-| DJANGO_SECRET_KEY                               | Secret key                                                                                                                  |                                                                         |
-| DJANGO_SERVER_TO_SERVER_API_TOKENS              |                                                                                                                             | []                                                                      |
-| DOCUMENT_IMAGE_MAX_SIZE                         | Maximum size of document in bytes                                                                                           | 10485760                                                                |
-| EMBEDDING_API_KEY                               | API key of the embedding api                                                                                                |                                                                         |
-| EMBEDDING_API_MODEL_NAME                        | Name of the embedding model used on the api                                                                                 | embeddings-small                                                        |
-| EMBEDDING_API_PATH                              | URL of the embedding api                                                                                                    |                                                                         |
-| EMBEDDING_DIMENSION                             | Size of the embedding vector                                                                                                | 1024                                                                    |
-| EMBEDDING_REQUEST_TIMEOUT                       | time out in seconds of the embedding requests                                                                               | 10                                                                      |
-| FRONTEND_CSS_URL                                | To add a external css file to the app                                                                                       |                                                                         |
-| FRONTEND_HOMEPAGE_FEATURE_ENABLED               | Frontend feature flag to display the homepage                                                                               | false                                                                   |
-| FRONTEND_THEME                                  | Frontend theme to use                                                                                                       |                                                                         |
-| HYBRID_SEARCH_ENABLED                           | Flag to enable hybrid (an then semantic) search                                                                             | True                                                                    |
-| HYBRID_SEARCH_WEIGHTS                           | Weights used in the weighted sum of the hybrid search score                                                                 | [0.3, 0.7]                                                              |
-| LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD         | Language detection confidence threshold                                                                                     | 0.75                                                                    |
-| LOGGING_LEVEL_LOGGERS_APP                       | Application logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"                                         | INFO                                                                    |
-| LOGGING_LEVEL_LOGGERS_ROOT                      | Default logging level. options are "DEBUG", "INFO", "WARN", "ERROR", "CRITICAL"                                             | INFO                                                                    |
-| LOGIN_REDIRECT_URL                              | Login redirect url                                                                                                          |                                                                         |
-| LOGIN_REDIRECT_URL_FAILURE                      | Login redirect url on failure                                                                                               |                                                                         |
-| LOGOUT_REDIRECT_URL                             | Logout redirect url                                                                                                         |                                                                         |
-| MALWARE_DETECTION_BACKEND                       | The malware detection backend use from the django-lasuite package                                                           | lasuite.malware_detection.backends.dummy.DummyBackend                   |
-| MALWARE_DETECTION_PARAMETERS                    | A dict containing all the parameters to initiate the malware detection backend                                              | {"callback_path": "core.malware_detection.malware_detection_callback",} |
-| MEDIA_BASE_URL                                  |                                                                                                                             |                                                                         |
-| NO_WEBSOCKET_CACHE_TIMEOUT                      | Cache used to store current editor session key when only users without websocket are editing a document                     | 120                                                                     |
-| OIDC_ALLOW_DUPLICATE_EMAILS                     | Allow duplicate emails                                                                                                      | false                                                                   |
-| OIDC_AUTH_REQUEST_EXTRA_PARAMS                  | OIDC extra auth parameters                                                                                                  | {}                                                                      |
-| OIDC_CREATE_USER                                | Create used on OIDC                                                                                                         | false                                                                   |
-| OIDC_DRF_AUTH_BACKEND                           | DRF Authentication backend class for OIDC                                                                                   | lasuite.oidc_login.backends.OIDCAuthenticationBackend                   |
-| OIDC_FALLBACK_TO_EMAIL_FOR_IDENTIFICATION       | Fallback to email for identification                                                                                        | true                                                                    |
-| OIDC_OP_AUTHORIZATION_ENDPOINT                  | Authorization endpoint for OIDC                                                                                             |                                                                         |
-| OIDC_OP_INTROSPECTION_ENDPOINT                  | Introspection endpoint for OIDC                                                                                             |                                                                         |
-| OIDC_OP_JWKS_ENDPOINT                           | JWKS endpoint for OIDC                                                                                                      |                                                                         |
-| OIDC_OP_LOGOUT_ENDPOINT                         | Logout endpoint for OIDC                                                                                                    |                                                                         |
-| OIDC_OP_TOKEN_ENDPOINT                          | Token endpoint for OIDC                                                                                                     |                                                                         |
-| OIDC_OP_USER_ENDPOINT                           | User endpoint for OIDC                                                                                                      |                                                                         |
-| OIDC_REDIRECT_ALLOWED_HOSTS                     | Allowed hosts for OIDC redirect url                                                                                         | []                                                                      |
-| OIDC_REDIRECT_REQUIRE_HTTPS                     | Require https for OIDC redirect url                                                                                         | false                                                                   |
-| OIDC_RP_CLIENT_ID                               | Client id used for OIDC                                                                                                     | impress                                                                 |
-| OIDC_RP_CLIENT_SECRET                           | Client secret used for OIDC                                                                                                 |                                                                         |
-| OIDC_RP_SCOPES                                  | Scopes requested for OIDC                                                                                                   | openid email                                                            |
-| OIDC_RP_SIGN_ALGO                               | verification algorithm used OIDC tokens                                                                                     | RS256                                                                   |
-| OIDC_RS_BACKEND_CLASS                           | Resource server backend class for OIDC                                                                                      | core.authentication.FinderResourceServerBackend                         |
-| OIDC_RS_AUDIENCE_CLAIM                          | The claim used to identify the audience                                                                                     | client_id                                                               |
-| OIDC_RS_CLIENT_ID                               | Client id used for OIDC resource server                                                                                     |                                                                         |
-| OIDC_RS_CLIENT_SECRET                           | Client secret used for OIDC resource server                                                                                 |                                                                         |
-| OIDC_RS_SIGNING_ALGO                            | Signing algorithm                                                                                                           | ES256                                                                   |
-| OIDC_RS_SCOPES                                  | Required scopes for authentication                                                                                          | ["openid"]                                                              |
-| OIDC_RS_PRIVATE_KEY_STR                         | Private key for encryption/decryption                                                                                       |                                                                         |
-| OIDC_RS_ENCRYPTION_KEY_TYPE                     | Encryption key type (RSA, EC, etc.)                                                                                         | RSA                                                                     |
-| OIDC_RS_ENCRYPTION_ALGO                         | Encryption algorithm                                                                                                        | RSA-OAEP                                                                |
-| OIDC_RS_ENCRYPTION_ENCODING                     | Encryption encoding algorithm                                                                                               | A256GCM                                                                 |
-| OIDC_STORE_ID_TOKEN                             | Store OIDC token                                                                                                            | true                                                                    |
-| OIDC_USE_NONCE                                  | Use nonce for OIDC                                                                                                          | true                                                                    |
-| OIDC_USERINFO_FULLNAME_FIELDS                   | OIDC token claims to create full name                                                                                       | ["first_name", "last_name"]                                             |
-| OIDC_USERINFO_SHORTNAME_FIELD                   | OIDC token claims to create shortname                                                                                       | first_name                                                              |
-| OIDC_VERIFY_SSL                                 | Enable SSL validation for OIDC                                                                                              | true                                                                    |
-| OIDC_TIMEOUT                                    | Timeout delay for OID requests                                                                                              |                                                                         |
-| OIDC_PROXY                                      | Defines a proxy for all requests to the OpenID Connect provider                                                             |                                                                         |
-| OPENSEARCH_HOST                                 | Opensearch database url                                                                                                     | default                                                                 |
-| OPENSEARCH_PORT                                 | Port of Opensearch database                                                                                                 | 9200                                                                    |
-| OPENSEARCH_USER                                 | Opensearch database user                                                                                                    | admin                                                                   |
-| OPENSEARCH_PASSWORD                             | Opensearch database user password                                                                                           |                                                                         |
-| OPENSEARCH_USE_SSL                              | Enable SSL connection for Opensearch database                                                                               | true                                                                    |
-| POSTHOG_KEY                                     | Posthog key for analytics                                                                                                   |                                                                         |
-| REDIS_URL                                       | Cache url                                                                                                                   | redis://redis:6379/1                                                    |
-| SENTRY_DSN                                      | Sentry host                                                                                                                 |                                                                         |
-| SESSION_COOKIE_AGE                              | duration of the cookie session                                                                                              | 60*60*12                                                                |
-| SPECTACULAR_SETTINGS_ENABLE_DJANGO_DEPLOY_CHECK |                                                                                                                             | false                                                                   |
-| STORAGES_STATICFILES_BACKEND                    |                                                                                                                             | whitenoise.storage.CompressedManifestStaticFilesStorage                 |
-| THEME_CUSTOMIZATION_CACHE_TIMEOUT               | Cache duration for the customization settings                                                                               | 86400                                                                   |
-| THEME_CUSTOMIZATION_FILE_PATH                   | Full path to the file customizing the theme. An example is provided in src/backend/impress/configuration/theme/default.json | BASE_DIR/impress/configuration/theme/default.json                       |
-| TRASHBIN_CUTOFF_DAYS                            | Trashbin cutoff                                                                                                             | 30                                                                      |
-| TRIGRAMS_BOOST                                  | weight boost applied to trigram score in document matching score                                                            | 0.25                                                                    |
-| TRIGRAMS_MINIMUM_SHOULD_MATCH                   | minimal number or proportion of trigrams having to match to score                                                           | 0.75%                                                                   |
-| USER_OIDC_ESSENTIAL_CLAIMS                      | Essential claims in OIDC token                                                                                              | []                                                                      |
-| Y_PROVIDER_API_BASE_URL                         | Y Provider url                                                                                                              |                                                                         |
-| Y_PROVIDER_API_KEY                              | Y provider API key                                                                                                          |                                                                         |

docs/evaluate-search-engine.md

@@ -1,32 +0,0 @@
-# Search Engine Evaluation Command
-
-## Overview 
-
-this Django command atomizes the evaluation of the search engine by computing 4 metrics: Average Discounted Cumulative Gain, Precision, Recall and F1 score. 
-
-## Usage 
-
-```
-python manage.py evaluate_search_engine <dataset_name> [options]
-```
-
-## Required Arguments
-- `dataset_name`: Name of the evaluation dataset to use. Datasets are located in `evaluation/management/commands/data/evaluation/`
-
-## Optional Arguments
-- `--min_score`: Minimum score threshold; hits below this score are ignored
-- `--keep-index`: Preserve the evaluation index after completion
-- `--force-reindex`: Drop and recreate the index even if it exists
-
-## Examples
-
-````
-# Basic evaluation with default settings
-python manage.py evaluate_search_engine my_dataset
-
-# Evaluation with minimum score threshold
-python manage.py evaluate_search_engine my_dataset --min_score 0.5
-
-# Force reindexing and clean up afterward
-python manage.py evaluate_search_engine my_dataset --force-reindex True --keep-index False
-````

docs/evaluation.md

@@ -1,170 +0,0 @@
-This file keeps for reference the logs of the last best evaluation of the model.
-
-````
-(venv) ➜  find git:(evaluate) ✗ docker compose exec app python manage.py evaluate_search_engine v1 --min_score 0.5   
-
-2025-11-20 18:44:30,903 core.services.opensearch INFO Hybrid search is disabled via HYBRID_SEARCH_ENABLED setting
-2025-11-20 18:44:31,183 core.management.commands.utils INFO Deleting search pipeline hybrid-search-pipeline
-2025-11-20 18:44:31,201 opensearch INFO DELETE http://opensearch:9200/_search/pipeline/hybrid-search-pipeline [status:200 request:0.018s]
-2025-11-20 18:44:31,202 opensearch WARNING GET http://opensearch:9200/_search/pipeline/hybrid-search-pipeline [status:404 request:0.001s]
-2025-11-20 18:44:31,202 core.management.commands.create_search_pipeline INFO Creating search pipeline: hybrid-search-pipeline
-2025-11-20 18:44:31,221 opensearch INFO PUT http://opensearch:9200/_search/pipeline/hybrid-search-pipeline [status:200 request:0.019s]
-2025-11-20 18:44:31,222 opensearch INFO HEAD http://opensearch:9200/evaluation-index [status:200 request:0.001s]
-[INFO] Starting evaluation with 76 documents and 12 queries
-2025-11-20 18:44:31,222 core.services.opensearch INFO embed: 'cours d'histoire de l'antiquité'
-2025-11-20 18:44:31,302 core.services.opensearch INFO Performing hybrid search with embedding: cours d'histoire de l'antiquité
-2025-11-20 18:44:31,317 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.014s]
-
-[QUERY EVALUATION]
-  q: cours d'histoire de l'antiquité
-  expect: ["L'Empire Romain", "L'Égypte Ancienne"]
-  result: ["L'Égypte Ancienne", 'La Sculpture sur Pierre']
-  NDCG: 61.31% 
-  PRECISION: 50.00% 
-  RECALL: 50.00% 
-  F1-SCORE: 50.00% 
-2025-11-20 18:44:31,317 core.services.opensearch INFO embed: 'recette salée végétarienne'
-2025-11-20 18:44:31,392 core.services.opensearch INFO Performing hybrid search with embedding: recette salée végétarienne
-2025-11-20 18:44:31,403 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.010s]
-
-[QUERY EVALUATION]
-  q: recette salée végétarienne
-  expect: ['Ratatouille Provençale', 'Salade de légumes', 'Fondue Savoyarde']
-  result: ['Salade de légumes']
-  NDCG: 46.93% 
-  PRECISION: 100.00% 
-  RECALL: 33.33% 
-  F1-SCORE: 50.00% 
-2025-11-20 18:44:31,403 core.services.opensearch INFO embed: 'art dramatique'
-2025-11-20 18:44:31,475 core.services.opensearch INFO Performing hybrid search with embedding: art dramatique
-2025-11-20 18:44:31,486 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.011s]
-
-[QUERY EVALUATION]
-  q: art dramatique
-  expect: ['Le Théâtre']
-  result: ['Le Théâtre', 'Le Vitrail']
-  NDCG: 100.00% 
-  PRECISION: 50.00% 
-  RECALL: 100.00% 
-  F1-SCORE: 66.67% 
-2025-11-20 18:44:31,487 core.services.opensearch INFO embed: 'art de bouger son corps'
-2025-11-20 18:44:31,556 core.services.opensearch INFO Performing hybrid search with embedding: art de bouger son corps
-2025-11-20 18:44:31,573 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.017s]
-
-[QUERY EVALUATION]
-  q: art de bouger son corps
-  expect: ['La Danse', 'La Danse Contemporaine']
-  result: ['La Danse', 'La Danse Contemporaine']
-  NDCG: 100.00% 
-  PRECISION: 100.00% 
-  RECALL: 100.00% 
-  F1-SCORE: 100.00% 
-2025-11-20 18:44:31,573 core.services.opensearch INFO embed: 'mammifères aquatiques'
-2025-11-20 18:44:31,641 core.services.opensearch INFO Performing hybrid search with embedding: mammifères aquatiques
-2025-11-20 18:44:31,654 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.013s]
-
-[QUERY EVALUATION]
-  q: mammifères aquatiques
-  expect: ['Le Dauphin', 'La Baleine à Bosse']
-  result: ['Le Dauphin', 'La Baleine à Bosse', 'Le Manchot Empereur', 'Le Requin Blanc', 'Le Paresseux']
-  NDCG: 100.00% 
-  PRECISION: 40.00% 
-  RECALL: 100.00% 
-  F1-SCORE: 57.14% 
-2025-11-20 18:44:31,654 core.services.opensearch INFO embed: 'insectes pollinisateurs'
-2025-11-20 18:44:31,733 core.services.opensearch INFO Performing hybrid search with embedding: insectes pollinisateurs
-2025-11-20 18:44:31,746 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.012s]
-
-[QUERY EVALUATION]
-  q: insectes pollinisateurs
-  expect: ["L'Abeille"]
-  result: ["L'Abeille", 'Le Caméléon', 'Le Papillon Monarque']
-  NDCG: 100.00% 
-  PRECISION: 33.33% 
-  RECALL: 100.00% 
-  F1-SCORE: 50.00% 
-2025-11-20 18:44:31,746 core.services.opensearch INFO embed: 'prédateur félin'
-2025-11-20 18:44:31,820 core.services.opensearch INFO Performing hybrid search with embedding: prédateur félin
-2025-11-20 18:44:31,834 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.014s]
-
-[QUERY EVALUATION]
-  q: prédateur félin
-  expect: ["Le Lion d'Afrique", 'Le Guépard']
-  result: ["Le Lion d'Afrique", 'Le Guépard', 'Le Requin Blanc', "L'éléphant", 'Le Hibou Grand-Duc', "L'Ours polaire", 'Le Serpent Python']
-  NDCG: 100.00% 
-  PRECISION: 28.57% 
-  RECALL: 100.00% 
-  F1-SCORE: 44.44% 
-2025-11-20 18:44:31,835 core.services.opensearch INFO embed: 'elephant'
-2025-11-20 18:44:31,906 core.services.opensearch INFO Performing hybrid search with embedding: elephant
-2025-11-20 18:44:31,920 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.013s]
-
-[QUERY EVALUATION]
-  q: elephant
-  expect: ["L'Éléphant d'Asie", "L'éléphant"]
-  result: ["L'éléphant", "L'Éléphant d'Asie"]
-  NDCG: 100.00% 
-  PRECISION: 100.00% 
-  RECALL: 100.00% 
-  F1-SCORE: 100.00% 
-2025-11-20 18:44:31,920 core.services.opensearch INFO embed: 'courir'
-2025-11-20 18:44:31,994 core.services.opensearch INFO Performing hybrid search with embedding: courir
-2025-11-20 18:44:32,010 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.015s]
-
-[QUERY EVALUATION]
-  q: courir
-  expect: ['Il va courir']
-  result: ['Il va courir']
-  NDCG: 100.00% 
-  PRECISION: 100.00% 
-  RECALL: 100.00% 
-  F1-SCORE: 100.00% 
-2025-11-20 18:44:32,011 core.services.opensearch INFO embed: 'football'
-2025-11-20 18:44:32,082 core.services.opensearch INFO Performing hybrid search with embedding: football
-2025-11-20 18:44:32,089 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.007s]
-
-[QUERY EVALUATION]
-  q: football
-  expect: ['Foot']
-  result: ['Foot']
-  NDCG: 100.00% 
-  PRECISION: 100.00% 
-  RECALL: 100.00% 
-  F1-SCORE: 100.00% 
-2025-11-20 18:44:32,089 core.services.opensearch INFO embed: 'couri'
-2025-11-20 18:44:32,156 core.services.opensearch INFO Performing hybrid search with embedding: couri
-2025-11-20 18:44:32,163 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.007s]
-
-[QUERY EVALUATION]
-  q: couri
-  expect: ['Il va courir']
-  result: ['Coq au Vin', 'Il va courir', 'Clafoutis aux Cerises']
-  NDCG: 63.09% 
-  PRECISION: 33.33% 
-  RECALL: 100.00% 
-  F1-SCORE: 50.00% 
-2025-11-20 18:44:32,164 core.services.opensearch INFO embed: 'courrir'
-2025-11-20 18:44:32,231 core.services.opensearch INFO Performing hybrid search with embedding: courrir
-2025-11-20 18:44:32,240 opensearch INFO POST http://opensearch:9200/evaluation-index/_search?search_pipeline=hybrid-search-pipeline&ignore_unavailable=true [status:200 request:0.009s]
-
-[QUERY EVALUATION]
-  q: courrir
-  expect: ['Il va courir']
-  result: ['Il va courir']
-  NDCG: 100.00% 
-  PRECISION: 100.00% 
-  RECALL: 100.00% 
-  F1-SCORE: 100.00% 
-
-============================================================
-[SUMMARY] Average Performance
-============================================================
-  Average NDCG: 89.28%
-  Average Precision: 69.60%
-  Average Recall: 90.28%
-  Average F1-Score: 72.35%
-2025-11-20 18:44:32,241 core.management.commands.utils INFO Deleting search pipeline hybrid-search-pipeline
-2025-11-20 18:44:32,258 opensearch INFO DELETE http://opensearch:9200/_search/pipeline/hybrid-search-pipeline [status:200 request:0.017s]
-
-[SUCCESS] Evaluation completed
-````

docs/release.md

@@ -1,58 +0,0 @@
-# Releasing a new version
-
-Whenever we are cooking a new release (e.g. `0.0.1`) we should follow a standard procedure described below:
-
-1.  Create a new branch named: `release/0.0.1`.
-
-2.  Bump the release number for backend project and Helm files:
-
-    - for backend, update the version number by hand in `pyproject.toml`,
-    - for Helm, update Docker image tag in files located at `src/helm/env.d` for both `preprod` and `production` environments:
-
-    ```yaml
-    image:
-    repository: lasuite/find
-    pullPolicy: Always
-    tag: "v0.0.1" # Replace with your new version number, without forgetting the "v" prefix
-    ...
-
-    The new images don't exist _yet_: they will be created automatically later in the process.
-
-3.  Update the project's `Changelog` following the [keepachangelog](https://keepachangelog.com/en/0.3.0/) recommendations
-
-4.  Commit your changes with the following format: the 🔖 release emoji, the type of release (patch/minor/patch) and the release version:
-
-    ```text
-    🔖(minor) bump release to 0.0.1
-    ```
-
-5.  Open a pull request, wait for an approval from your peers and merge it.
-6.  Checkout and pull changes from the `main` branch to ensure you have the latest updates.
-7.  Tag and push your commit:
-
-    ```bash
-    git tag v0.0.1 && git push origin tag v0.0.1
-    ```
-
-    Doing this triggers the CI and tells it to build the new Docker image versions that you targeted earlier in the Helm files.
-
-8.  Ensure the new [backend](https://hub.docker.com/r/lasuite/find/tags) image tags are on Docker Hub.
-9.  The release is now done!
-
-# Deploying
-
-> [!TIP]
-> The `staging` platform is deployed automatically with every update of the `main` branch.
-
-Making a new release doesn't publish it automatically in production.
-
-Deployment is done by ArgoCD. ArgoCD checks for the `production` tag and automatically deploys the production platform with the targeted commit.
-
-To publish, we mark the commit we want with the `production` tag. ArgoCD is then notified that the tag has changed. It then deploys the Docker image tags specified in the Helm files of the targeted commit.
-
-To publish the release you just made:
-
-```bash
-git tag --force production v0.0.1
-git push --force origin production
-```

docs/setup-for-docs.md

@@ -1,39 +0,0 @@
-# Setup the Find search for Impress
-
-This configuration will enable the fulltext search feature for Docs :
-- Each save on **core.Document** or **core.DocumentAccess** will trigger the indexer
-- The `api/v1.0/documents/search/` will work as a proxy with the Find API for fulltext search.
-
-## Create an index service for Docs
-
-Configure a **Service** for Docs application with these settings
-
-- **Name**: `docs`<br>_request.auth.name of the Docs application._
-- **Client id**: `impress`<br>_Name of the token audience or client_id of the Docs application._
-
-See [how-to-use-indexer.md](how-to-use-indexer.md) for details.
-
-## Configure settings of Docs
-
-Add those Django settings the Docs application to enable the feature.
-
-```shell
-SEARCH_INDEXER_CLASS="core.services.search_indexers.FindDocumentIndexer"
-SEARCH_INDEXER_COUNTDOWN=10  # Debounce delay in seconds for the indexer calls.
-
-# The token from service "docs" of Find application (development).
-SEARCH_INDEXER_SECRET="find-api-key-for-docs-with-exactly-50-chars-length"
-SEARCH_INDEXER_URL="http://find:8000/api/v1.0/documents/index/"
-
-# Search endpoint. Uses the OIDC token for authentication
-SEARCH_INDEXER_QUERY_URL="http://find:8000/api/v1.0/documents/search/"
-```
-
-We also need to enable the **OIDC Token** refresh or the authentication will fail quickly.
-
-```shell
-# Store OIDC tokens in the session
-OIDC_STORE_ACCESS_TOKEN = True  # Store the access token in the session
-OIDC_STORE_REFRESH_TOKEN = True  # Store the encrypted refresh token in the session
-OIDC_STORE_REFRESH_TOKEN_KEY = "your-32-byte-encryption-key=="  # Must be a valid Fernet key (32 url-safe base64-encoded bytes)
-```

docs/setup-for-drive.md

@@ -1,43 +0,0 @@
-# Setup the Find search for Drive
-
-This configuration will enable the fulltext search feature for Docs :
-- Each save on **core.Item** or **core.Item** will trigger the indexer
-- Once indexer service configured, the `api/v1.0/item/search/` will work as a proxy with the Find API for fulltext search.
-
-## Create an index service for Drive
-
-Configure a **Service** for Docs application with these settings
-
-- **Name**: `drive`<br>_request.auth.name of the Docs application._
-- **Client id**: `drive`<br>_Name of the token audience or client_id of the Docs application._
-
-See [how-to-use-indexer.md](how-to-use-indexer.md) for details.
-
-## Configure settings of Drive
-
-Add those Django settings the Docs application to enable the feature.
-
-```python
-SEARCH_INDEXER_CLASS="core.services.search_indexers.SearchIndexer"
-SEARCH_INDEXER_COUNTDOWN=10  # Debounce delay in seconds for the indexer calls.
-
-# The token from service "drive" of Find application (development)
-SEARCH_INDEXER_SECRET=find-api-key-for-driv-with-exactly-50-chars-length
-SEARCH_INDEXER_URL="http://find:8000/api/v1.0/documents/index/"
-
-# Search endpoint. Uses the OIDC token for authentication
-SEARCH_INDEXER_QUERY_URL="http://find:8000/api/v1.0/documents/search/"
-
-# Limit the mimetypes and size of indexable files
-SEARCH_INDEXER_ALLOWED_MIMETYPES=["text/"]
-SEARCH_INDEXER_UPLOAD_MAX_SIZE=2 * 2**20  # 2Mb
-```
-
-We also need to enable the **OIDC Token** refresh or the authentication will fail quickly.
-
-```shell
-# Store OIDC tokens in the session
-OIDC_STORE_ACCESS_TOKEN = True  # Store the access token in the session
-OIDC_STORE_REFRESH_TOKEN = True  # Store the encrypted refresh token in the session
-OIDC_STORE_REFRESH_TOKEN_KEY = "your-32-byte-encryption-key=="  # Must be a valid Fernet key (32 url-safe base64-encoded bytes)
-```

docs/setup-indexer.md

@@ -1,161 +0,0 @@
-# Using the Find indexer
-
-This guide explains how to setup the Find service which provide an API for indexation and fulltext search of
-documents from various sources in a secure way : only the documents within the scope of the user's OIDC token
-are visible.
-
-## Setup Opensearch
-
-### General
-
-Add the following variables to your Django settings to configure Find and enable full-text search.
-
-```python
-# Login for opensearch
-OPENSEARCH_USER=opensearch-user
-OPENSEARCH_PASSWORD=your-opensearch-password
-
-# Host configuration
-OPENSEARCH_HOST=opensearch
-OPENSEARCH_PORT=9200
-
-# Enable SSL for opensearch connection (False in dev mode)
-OPENSEARCH_USE_SSL=True
-
-# Prefix for the index name of the registered services.
-OPENSEARCH_INDEX_PREFIX=find
-```
-
-### Language
-
-Language specific operations are applied to document titles and contents to improve search results. 
-The language is automatically detected  by Find.
-If the language can not be detected no language specific operation are applied and the indexing process is not affected.
-
-Find supports french, english, german and dutch. 
-
-The search process is not language specific, a query can get documents of any language.
-
-Language detection estimates a confidence between 0 and 1. If the confidence is below a threshold the language is considered unrecognized. 
-This threshold can be controlled with LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD environment variable.
-
-```python
-LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD=0.75
-```
-
-### Semantic search
-
-Find offers a semantic search feature. You can either use pure full-text search or a hybrid full-text + semantic search. To enable the hybrid search, add the following settings. 
-
-```python
-# Enable flag
-HYBRID_SEARCH_ENABLED = True
-
-# weighted sum: full_text_weight, semantic_search_weight
-HYBRID_SEARCH_WEIGHTS = 0.7,0.3
-
-# Embedding
-CHUNK_SIZE=512
-CHUNK_OVERLAP=50
-EMBEDDING_API_PATH = https://embedding.api.example.com/full/path/
-EMBEDDING_API_KEY = your-embedding-api-key
-EMBEDDING_REQUEST_TIMEOUT = 10
-EMBEDDING_API_MODEL_NAME = embedding-api-model-name
-EMBEDDING_DIMENSION = 1024
-```
-
-The hybrid search computes a score for full-text and semantic search and combines them through a weighted sum. HYBRID_SEARCH_WEIGHTS contains the weights of full-text and semantic respectively. 
-
-You need to use an embedding api similar to https://albert.api.etalab.gouv.fr/documentation#tag/Embeddings/operation/embeddings_v1_embeddings_post. 
-
-### document chunking
-
-The indexing process embeds documents by converting their content into vector representations (embeddings). When a document exceeds the character dimension defined by CHUNK_SIZE, it's divided into smaller segments (chunks), with each chunk embedded independently. Each chunk must be smaller than the embedding model's context window . 
-
-The chunking algorithm works recursively. It attempts to create the largest possible segments first, then subdivides them further if they still exceed the size limit defined by CHUNK_SIZE. Segments can share overlapping content between them (set CHUNK_OVERLAP=0 to disable overlapping). 
-
-During the search, the matching of a document is the matching of its best chunk.
-
-## trigrams
-
-Find uses trigrams to improve the robustness of the full text search engine to spelling variations and errors. It can be configured by two environment variables. 
-
-````
-TRIGRAMS_BOOST=0.25
-TRIGRAMS_MINIMUM_SHOULD_MATCH=0.75%
-````
-
-`TRIGRAMS_BOOST` is weight boost applied to the trigram score in the document matching. 
-`TRIGRAMS_MINIMUM_SHOULD_MATCH` is the minimal number or proportion of trigrams having to match to score. It is
-either an absolute number or proportion.
-
-## Setup indexation API
-
-Other applications can index their files through the **`/index/`** endpoint with a simple token authentication.
-
-For each application a new **Service** must be created through the admin interface
-(see http://localhost:9071/admin/core/service/add/)
-
-| Field                       | Description                                        |
-|-----------------------------|----------------------------------------------------|
-| Name                        | Name of the service and also the name of the index in Opensearch database |
-| Is active                   | Toggle service availability                        |
-| Client id                   | Calling service client_id (e.g `impress` for docs) |
-| Allowed services for search | List of sub-services. Will add the results from all these index<br>to the search results. |
-| Token (_read-only_)         | Random token for calling service authentication    |
-
-And add the key in the calling application Django settings.
-
-**Development Mode (Docs + Find)**
-
-The command `make demo` will create a working service configuration for `docs` and `drive` with predefined secret keys
-
-```python
-# Docs
-SEARCH_INDEXER_SECRET="find-api-key-for-docs-with-exactly-50-chars-length"
-```
-
-```python
-# Drive
-SEARCH_INDEXER_SECRET="find-api-key-for-driv-with-exactly-50-chars-length"
-```
-
-## Setup search API
-
-The **`/search/`** endpoint is an OIDC ResourceServer view and needs extra Django settings (see [lasuite](https://github.com/suitenumerique/django-lasuite/blob/main/documentation/how-to-use-oidc-resource-server-backend.md) for details)
-
-```shell
-OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs
-OIDC_OP_AUTHORIZATION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/auth
-OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token
-OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/userinfo
-
-# To run Find in development mode along other projects like docs/impress
-# we should to use OIDC endpoints on a common keycloak realm. e.g :
-# OIDC_OP_URL = http://nginx:8083/realms/impress
-#
-# This will cause a conflict with the 'iss' claim validation rule because the docs realm
-# gives {'iss': 'http://localhost:8083/realms/impress'} so it must be the same
-OIDC_OP_URL=http://localhost:8083/realms/impress
-
-# Introspection endpoint is needed to get the "audience" and "sub" from the user token
-OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token/introspect
-
-# In development, the resource server use insecure settings
-# OIDC_VERIFY_SSL=False
-
-# Resource server
-OIDC_RS_SCOPES="openid"
-OIDC_RS_SIGN_ALGO=RS256
-
-# This backend allows authentication without any model in database. 
-OIDC_RS_BACKEND_CLASS="core.authentication.FinderResourceServerBackend"
-```
-
-**Development mode (Docs + Find)**
-
-Docs and Find projects stacks can be run together but must have the same keycloak server.
-
-So the endpoints of Docs on the 'nginx' domain for ResourceServer authentication and introspection are also used for Find.
-
-**IMPORTANT:** Keep OIDC_OP_URL on 'localhost' or it will break the OIDC token claims validation : the `iss` claim from the token of the 'impress' users are 'localhost' and not 'nginx'.

docs/system-requirements.md

@@ -1,12 +0,0 @@
-# La Suite Find – System & Requirements
-
-## Ports (dev defaults)
-
-| Port      | Service               |
-| --------- | --------------------- |
-| 8081      | Django (main)         |
-| 8083      | Nginx proxy           |
-| 25432     | PostgreSQL (main)     |
-| 9200      | Opensearch            |
-| 9600      | Opensearch admin      |
-| 5601      | Opensearch dashboard  |

env.d/development/common.dist

@@ -15,23 +15,15 @@ FIND_BASE_URL="http://localhost:8072"
 # Opensearch
 OPENSEARCH_PASSWORD=find.PASS123
 OPENSEARCH_USE_SSL=false
-OPENSEARCH_INDEX_PREFIX=find
 
 # OIDC
 OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/certs
-OIDC_OP_AUTHORIZATION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/auth
+OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/impress/protocol/openid-connect/auth
 OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token
 OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/userinfo
+OIDC_OP_URL=http://nginx:8083/realms/impress
 OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token/introspect
 
-# To run Find in development mode along other projects like docs/impress
-# we should to use OIDC endpoints on a common keycloak realm. e.g :
-# OIDC_OP_URL = http://nginx:8083/realms/impress
-#
-# This will cause a conflict with the 'iss' claim validation rule because the docs realm
-# gives {'iss': 'http://localhost:8083/realms/impress'} so it must be the same
-OIDC_OP_URL=http://localhost:8083/realms/impress
-
 OIDC_RP_CLIENT_ID=find
 OIDC_RP_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly
 OIDC_RP_SIGN_ALGO=RS256
@@ -50,12 +42,3 @@ OIDC_RS_SIGN_ALGO=RS256
 
 OIDC_RS_BACKEND_CLASS="core.authentication.FinderResourceServerBackend"
 OIDC_RS_ENCRYPTION_KEY_TYPE="RSA"
-
-# Hybrid Search settings
-HYBRID_SEARCH_ENABLED=True
-EMBEDDING_API_KEY=ThisIsAnExampleKeyForDevPurposeOnly
-EMBEDDING_API_PATH=https://albert.api.etalab.gouv.fr/v1/embeddings
-
-## Multi-embedding: chunk documents and embed each chunk
-CHUNK_SIZE=512
-CHUNK_OVERLAP=50

src/backend/.pylintrc

@@ -7,7 +7,7 @@ extension-pkg-whitelist=
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=migrations,.venv
+ignore=migrations
 
 # Add files or directories matching the regex patterns to the blacklist. The
 # regex matches against base names, not paths.
@@ -31,6 +31,10 @@ persistent=yes
 # Specify a configuration file.
 #rcfile=
 
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages
+suggestion-mode=yes
+
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no

src/backend/core/admin.py

@@ -1,16 +1,8 @@
 """Admin config for find's core app"""
 
-from django.contrib import admin, messages
-from django.shortcuts import redirect, render
-from django.urls import path, reverse
+from django.contrib import admin
 
-from core.management.commands.create_search_pipeline import (
-    ensure_search_pipeline_exists,
-)
-
-from . import selftests_builtin  # pylint: disable=unused-import
 from .models import Service
-from .selftests import registry
 
 
 @admin.register(Service)
@@ -22,79 +14,3 @@ class ServiceAdmin(admin.ModelAdmin):
     list_filter = ("is_active", "created_at")
     ordering = ("-created_at",)
     readonly_fields = ("created_at", "token")
-    change_list_template = "admin/core/services/change_list.html"
-
-    def get_urls(self):
-        urls = super().get_urls()
-        custom_urls = [
-            path(
-                "ensure-search-pipeline/",
-                self.admin_site.admin_view(self.ensure_search_pipeline_view),
-                name="core_service_ensure_search_pipeline",
-            ),
-        ]
-        return custom_urls + urls
-
-    def ensure_search_pipeline_view(self, request):
-        """Run the management command function to assert the pipeline exists."""
-        changelist_url = reverse("admin:core_service_changelist")
-
-        try:
-            ensure_search_pipeline_exists()
-        except Exception as exc:  # noqa: BLE001# pylint: disable=broad-exception-caught
-            self.message_user(
-                request, f"Failed to ensure search pipeline: {exc}", messages.ERROR
-            )
-        else:
-            self.message_user(
-                request, "Search pipeline presence insured", messages.SUCCESS
-            )
-
-        return redirect(changelist_url)
-
-
-def selftest_view(request):
-    """Display the self-test page and run tests if requested."""
-    # selftests_builtin and registry are imported at module level to ensure tests are registered
-    run_tests = request.GET.get("run", "false").lower() == "true"
-
-    if run_tests:
-        results = registry.run_all()
-        all_passed = all(result.success for result in results)
-    else:
-        results = []
-        all_passed = None
-
-    context = {
-        **admin.site.each_context(request),
-        "title": "System Self-Tests",
-        "results": results,
-        "all_passed": all_passed,
-        "run_tests": run_tests,
-        "available_tests": registry.get_all_tests(),
-    }
-
-    return render(request, "admin/selftest.html", context)
-
-
-# Add custom URL to the default admin site
-def get_admin_urls():
-    """Get URLs with selftest added."""
-
-    def get_urls():
-        urls = get_admin_urls.original_get_urls()
-        custom_urls = [
-            path(
-                "selftest/",
-                admin.site.admin_view(selftest_view),
-                name="selftest",
-            ),
-        ]
-        return custom_urls + urls
-
-    return get_urls
-
-
-# Store original get_urls and override it
-get_admin_urls.original_get_urls = admin.site.get_urls
-admin.site.get_urls = get_admin_urls()

src/backend/core/apps.py

@@ -1,26 +0,0 @@
-"""Find Core application"""
-
-from django.apps import AppConfig
-from django.utils.translation import gettext_lazy as _
-
-from core.management.commands.create_search_pipeline import (
-    ensure_search_pipeline_exists,
-)
-from core.services.opensearch import (
-    check_hybrid_search_enabled,
-)
-
-
-class CoreConfig(AppConfig):
-    """Configuration class for the Find core app."""
-
-    name = "core"
-    app_label = "core"
-    verbose_name = _("Find core application")
-
-    def ready(self):
-        """
-        Ensure search pipeline exists if hybrid search is enabled.
-        """
-        if check_hybrid_search_enabled():
-            ensure_search_pipeline_exists()

src/backend/core/enums.py

@@ -13,16 +13,6 @@ class ReachEnum(str, Enum):
     RESTRICTED = "restricted"
 
 
-# Search type
-
-
-class SearchTypeEnum(str, Enum):
-    """Search type options"""
-
-    HYBRID = "hybrid"
-    FULL_TEXT = "full_text"
-
-
 # Fields
 
 CREATED_AT = "created_at"
@@ -31,9 +21,7 @@ PATH = "path"
 NUMCHILD = "numchild"
 REACH = "reach"
 SIZE = "size"
-TAGS = "tags"
 TITLE = "title"
-CONTENT = "content"
 UPDATED_AT = "updated_at"
 USERS = "users"
 GROUPS = "groups"
@@ -41,15 +29,4 @@ GROUPS = "groups"
 RELEVANCE = "relevance"
 
 ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, REACH)
-SOURCE_FIELDS = (
-    TITLE,
-    CONTENT,
-    SIZE,
-    DEPTH,
-    PATH,
-    NUMCHILD,
-    CREATED_AT,
-    UPDATED_AT,
-    REACH,
-    TAGS,
-)
+SOURCE_FIELDS = (TITLE, SIZE, DEPTH, PATH, NUMCHILD, CREATED_AT, UPDATED_AT, REACH)

src/backend/core/factories.py

@@ -30,7 +30,6 @@ class DocumentSchemaFactory(factory.DictFactory):
     users = factory.LazyFunction(lambda: [str(uuid4()) for _ in range(3)])
     groups = factory.LazyFunction(lambda: [slugify(fake.word()) for _ in range(3)])
     reach = factory.Iterator(list(enums.ReachEnum))
-    tags = factory.LazyFunction(lambda: [])
     depth = 1
     numchild = 0
     is_active = True

src/backend/core/management/commands/create_search_pipeline.py

@@ -1,54 +0,0 @@
-"""
-Handle create the search pipeline command of the hybrid search.
-"""
-
-import logging
-
-from django.conf import settings
-from django.core.management.base import BaseCommand
-
-from opensearchpy.exceptions import NotFoundError
-
-from core.services.opensearch import (
-    opensearch_client,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    """Handle create the search pipeline command of the hybrid search."""
-
-    help = __doc__
-
-    def handle(self, *args, **options):
-        ensure_search_pipeline_exists()
-
-
-def ensure_search_pipeline_exists():
-    """Create search pipeline for hybrid search if it does not exist"""
-    try:
-        opensearch_client().search_pipeline.get(id=settings.HYBRID_SEARCH_PIPELINE_ID)
-        logger.info("Search pipeline exists already")
-    except NotFoundError:
-        logger.info("Creating search pipeline: %s", settings.HYBRID_SEARCH_PIPELINE_ID)
-        opensearch_client().transport.perform_request(
-            method="PUT",
-            url="/_search/pipeline/" + settings.HYBRID_SEARCH_PIPELINE_ID,
-            body={
-                "description": "Post processor for hybrid search",
-                "phase_results_processors": [
-                    {
-                        "normalization-processor": {
-                            "normalization": {"technique": "min_max"},
-                            "combination": {
-                                "technique": "arithmetic_mean",
-                                "parameters": {
-                                    "weights": settings.HYBRID_SEARCH_WEIGHTS
-                                },
-                            },
-                        }
-                    }
-                ],
-            },
-        )

src/backend/core/management/commands/reindex_with_embedding.py

@@ -1,141 +0,0 @@
-"""
-Handle reindexing of documents with embeddings in OpenSearch.
-"""
-
-import logging
-
-from django.conf import settings
-from django.core.management.base import BaseCommand, CommandError
-
-from opensearchpy.exceptions import NotFoundError
-
-from core.models import get_opensearch_index_name
-from core.services.indexing import chunk_document
-from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
-from core.utils import get_language_value
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    """Reindex all documents with embeddings"""
-
-    help = __doc__
-    opensearch_client_ = opensearch_client()
-
-    def add_arguments(self, parser):
-        parser.add_argument("index_name", type=str)
-
-    def handle(self, *args, **options):
-        """Launch the reindexing with embedding."""
-
-        index_name = get_opensearch_index_name(options["index_name"])
-
-        if not check_hybrid_search_enabled():
-            raise CommandError("Hybrid search is not enabled or properly configured.")
-
-        try:
-            self.opensearch_client_.indices.get(index=index_name)
-        except NotFoundError as error:
-            raise CommandError(f"Index {index_name} does not exist.") from error
-
-        self.stdout.write(f"[INFO] Start reindexing {index_name} with embedding.")
-
-        result = reindex_with_embedding(index_name)
-
-        self.stdout.write(
-            f"[INFO] Reindexing of {index_name} is done.\n"
-            f"nb success embedding: {result['nb_success_embedding']}\n"
-            f"nb failed embedding: {result['nb_failed_embedding']} embedding fails\n"
-        )
-
-
-def reindex_with_embedding(index_name, batch_size=500, scroll="10m"):
-    """
-    Reindex documents from source index to destination index with embeddings.
-
-    returns a dict with the number of successful embeddings and failed embeddings.
-    """
-    opensearch_client_ = opensearch_client()
-    page = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
-        index=index_name,
-        scroll=scroll,
-        size=batch_size,
-        seq_no_primary_term=True,
-        body={
-            "query": {
-                "bool": {
-                    "should": [
-                        {
-                            "bool": {
-                                "must_not": [
-                                    {
-                                        "nested": {
-                                            "path": "chunks",
-                                            "query": {"match_all": {}},
-                                        }
-                                    }
-                                ]
-                            }
-                        },
-                        {
-                            "bool": {
-                                "must_not": [
-                                    {
-                                        "term": {
-                                            "embedding_model": settings.EMBEDDING_API_MODEL_NAME
-                                        }
-                                    }
-                                ]
-                            }
-                        },
-                    ],
-                    "minimum_should_match": 1,
-                }
-            }
-        },
-    )
-    nb_failed_embedding = 0
-    nb_success_embedding = 0
-    while len(page["hits"]["hits"]) > 0:
-        actions = []
-        for hit in page["hits"]["hits"]:
-            source = hit["_source"]
-            chunks = chunk_document(
-                get_language_value(source, "title"),
-                get_language_value(source, "content"),
-            )
-            if chunks:
-                actions.append(
-                    {
-                        "update": {
-                            "_id": hit["_id"],
-                            # if_seq_no and if_primary_term ensure we only update indexes
-                            # if the document hasn't changed
-                            "if_seq_no": hit["_seq_no"],
-                            "if_primary_term": hit["_primary_term"],
-                        }
-                    }
-                )
-                actions.append(
-                    {
-                        "doc": {
-                            "chunks": chunks,
-                            "embedding_model": settings.EMBEDDING_API_MODEL_NAME,
-                        }
-                    }
-                )
-                nb_success_embedding += 1
-            else:
-                nb_failed_embedding += 1
-
-        opensearch_client_.bulk(index=index_name, body=actions)
-        page = opensearch_client_.scroll(  # pylint: disable=unexpected-keyword-arg
-            scroll_id=page["_scroll_id"], scroll=scroll
-        )
-
-    opensearch_client_.clear_scroll(scroll_id=page["_scroll_id"])
-    return {
-        "nb_failed_embedding": nb_failed_embedding,
-        "nb_success_embedding": nb_success_embedding,
-    }

src/backend/core/models.py

@@ -3,11 +3,9 @@
 import secrets
 import string
 
-from django.conf import settings
 from django.contrib.auth.models import AbstractUser
 from django.db import models
 from django.db.models.functions import Length
-from django.utils.functional import cached_property
 from django.utils.text import slugify
 from django.utils.translation import gettext_lazy as _
 
@@ -15,11 +13,6 @@ models.CharField.register_lookup(Length)
 TOKEN_LENGTH = 50
 
 
-def get_opensearch_index_name(name: str):
-    """Returns the opensearch index for a service name"""
-    return f"{settings.OPENSEARCH_INDEX_PREFIX}-{name}"
-
-
 class User(AbstractUser):
     """User for the find application"""
 
@@ -68,8 +61,3 @@ class Service(models.Model):
         )
         token = "".join(secrets.choice(characters) for _ in range(TOKEN_LENGTH))
         return token
-
-    @cached_property
-    def index_name(self):
-        """Returns the opensearch index for the service"""
-        return get_opensearch_index_name(self.name)

src/backend/core/opensearch.py

@@ -0,0 +1,54 @@
+"""Opensearch related utils."""
+
+from django.conf import settings
+
+from opensearchpy import OpenSearch
+from opensearchpy.exceptions import NotFoundError
+
+client = OpenSearch(
+    hosts=[{"host": settings.OPENSEARCH_HOST, "port": settings.OPENSEARCH_PORT}],
+    http_auth=(settings.OPENSEARCH_USER, settings.OPENSEARCH_PASSWORD),
+    timeout=50,
+    use_ssl=settings.OPENSEARCH_USE_SSL,
+    verify_certs=False,
+)
+
+
+def ensure_index_exists(index_name):
+    """Create index if it does not exist"""
+    try:
+        client.indices.get(index=index_name)
+    except NotFoundError:
+        client.indices.create(
+            index=index_name,
+            body={
+                "mappings": {
+                    "dynamic": "strict",
+                    "properties": {
+                        "id": {"type": "keyword"},
+                        "title": {
+                            "type": "keyword",  # Primary field for exact matches and sorting
+                            "fields": {
+                                "text": {
+                                    "type": "text"
+                                }  # Sub-field for full-text search
+                            },
+                        },
+                        "depth": {"type": "integer"},
+                        "path": {
+                            "type": "keyword",
+                            "fields": {"text": {"type": "text"}},
+                        },
+                        "numchild": {"type": "integer"},
+                        "content": {"type": "text"},
+                        "created_at": {"type": "date"},
+                        "updated_at": {"type": "date"},
+                        "size": {"type": "long"},
+                        "users": {"type": "keyword"},
+                        "groups": {"type": "keyword"},
+                        "reach": {"type": "keyword"},
+                        "is_active": {"type": "boolean"},
+                    },
+                }
+            },
+        )

src/backend/core/schemas.py

@@ -18,7 +18,6 @@ from pydantic import (
 )
 
 from . import enums
-from .enums import SearchTypeEnum
 
 
 class DocumentSchema(BaseModel):
@@ -38,7 +37,6 @@ class DocumentSchema(BaseModel):
         default_factory=list
     )
     reach: Optional[enums.ReachEnum] = Field(default=enums.ReachEnum.RESTRICTED)
-    tags: List[Annotated[str, Field(max_length=100)]] = Field(default_factory=list)
     is_active: bool
 
     model_config = ConfigDict(
@@ -114,26 +112,7 @@ class SearchQueryParametersSchema(BaseModel):
     services: StringListParameter = Field(default_factory=list)
     visited: StringListParameter = Field(default_factory=list)
     reach: Optional[enums.ReachEnum] = None
-    tags: StringListParameter = Field(default_factory=list)
-    path: Optional[str] = None
     order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default=enums.RELEVANCE)
     order_direction: Optional[Literal["asc", "desc"]] = Field(default="desc")
-    nb_results: Optional[conint(ge=1, le=300)] = Field(default=50)
-    search_type: Optional[SearchTypeEnum] = Field(default=None)
-
-
-class DeleteDocumentsSchema(BaseModel):
-    """Schema for validating the delete documents request"""
-
-    service: str = Field(max_length=300)
-    document_ids: Optional[List[str]] = Field(default=None)
-    tags: Optional[List[str]] = Field(default=None)
-
-    @model_validator(mode="after")
-    def check_at_least_one_filter(self):
-        """Ensure at least one of document_ids or tags is provided"""
-        if not self.document_ids and not self.tags:
-            raise ValueError(
-                "At least one of 'document_ids' or 'tags' must be provided"
-            )
-        return self
+    page_number: Optional[conint(ge=1)] = Field(default=1)
+    page_size: Optional[conint(ge=1, le=100)] = Field(default=50)

src/backend/core/selftests.py

@@ -1,125 +0,0 @@
-"""
-Selftest registry and base classes for system health checks.
-
-This module provides a modular system for registering and running self-tests
-that check the health of various system components.
-"""
-
-import logging
-from typing import Dict, List, Optional
-
-logger = logging.getLogger(__name__)
-
-
-class SelfTestResult:
-    """Result of a self-test execution."""
-
-    def __init__(  # pylint: disable=too-many-arguments,too-many-positional-arguments
-        self,
-        name: str,
-        success: bool,
-        message: str,
-        details: Optional[Dict] = None,
-        duration_ms: Optional[float] = None,
-    ):
-        self.name = name
-        self.success = success
-        self.message = message
-        self.details = details or {}
-        self.duration_ms = duration_ms
-
-    def to_dict(self) -> Dict:
-        """Convert result to dictionary."""
-        return {
-            "name": self.name,
-            "success": self.success,
-            "message": self.message,
-            "details": self.details,
-            "duration_ms": self.duration_ms,
-        }
-
-
-class SelfTest:
-    """Base class for self-tests."""
-
-    name: str = "Base Self Test"
-    description: str = "Base self-test class"
-
-    def run(self) -> SelfTestResult:
-        """
-        Execute the self-test.
-
-        Returns:
-            SelfTestResult: The result of the test execution.
-        """
-        raise NotImplementedError("Subclasses must implement the run method")
-
-
-class SelfTestRegistry:
-    """Registry for managing self-tests."""
-
-    def __init__(self):
-        self._tests: Dict[str, SelfTest] = {}
-
-    def register(self, test_class: type[SelfTest]) -> None:
-        """
-        Register a self-test class.
-
-        Args:
-            test_class: The SelfTest subclass to register.
-        """
-        test_instance = test_class()
-        test_id = test_class.__name__
-        if test_id in self._tests:
-            logger.warning("Self-test %s is already registered. Overwriting.", test_id)
-        self._tests[test_id] = test_instance
-        logger.debug("Registered self-test: %s - %s", test_id, test_instance.name)
-
-    def unregister(self, test_class: type[SelfTest]) -> None:
-        """
-        Unregister a self-test class.
-
-        Args:
-            test_class: The SelfTest subclass to unregister.
-        """
-        test_id = test_class.__name__
-        if test_id in self._tests:
-            del self._tests[test_id]
-            logger.debug("Unregistered self-test: %s", test_id)
-
-    def get_all_tests(self) -> List[SelfTest]:
-        """
-        Get all registered tests.
-
-        Returns:
-            List of registered SelfTest instances.
-        """
-        return list(self._tests.values())
-
-    def run_all(self) -> List[SelfTestResult]:
-        """
-        Run all registered tests.
-
-        Returns:
-            List of SelfTestResult objects.
-        """
-        results = []
-        for test in self._tests.values():
-            try:
-                result = test.run()
-                results.append(result)
-            except Exception as e:  # pylint: disable=broad-exception-caught
-                logger.exception("Error running self-test %s: %s", test.name, e)
-                results.append(
-                    SelfTestResult(
-                        name=test.name,
-                        success=False,
-                        message=f"Test failed with exception: {str(e)}",
-                        details={"exception": str(e)},
-                    )
-                )
-        return results
-
-
-# Global registry instance
-registry = SelfTestRegistry()

src/backend/core/selftests_builtin.py

@@ -1,236 +0,0 @@
-"""
-Built-in self-tests for core system components.
-
-This module contains self-tests for database, cache, and OpenSearch connectivity.
-"""
-
-import time
-
-from django.conf import settings
-from django.core.cache import cache
-from django.db import connection
-
-import sentry_sdk
-
-from .selftests import SelfTest, SelfTestResult, registry
-from .services.opensearch import opensearch_client
-
-
-class DatabaseSelfTest(SelfTest):
-    """Test database connectivity."""
-
-    name = "Database Connection"
-    description = "Verify that the database is accessible and responsive"
-
-    def run(self) -> SelfTestResult:
-        """Test database connection by executing a simple query."""
-        start_time = time.time()
-        try:
-            with connection.cursor() as cursor:
-                cursor.execute("SELECT 1")
-                result = cursor.fetchone()
-
-            duration_ms = (time.time() - start_time) * 1000
-
-            if result and result[0] == 1:
-                return SelfTestResult(
-                    name=self.name,
-                    success=True,
-                    message="Database connection successful",
-                    details={
-                        "database": settings.DATABASES["default"]["NAME"],
-                        "engine": settings.DATABASES["default"]["ENGINE"],
-                    },
-                    duration_ms=duration_ms,
-                )
-            return SelfTestResult(
-                name=self.name,
-                success=False,
-                message="Database query returned unexpected result",
-                duration_ms=duration_ms,
-            )
-        except (OSError, ValueError) as e:
-            duration_ms = (time.time() - start_time) * 1000
-            return SelfTestResult(
-                name=self.name,
-                success=False,
-                message=f"Database connection failed: {str(e)}",
-                details={"exception": str(e)},
-                duration_ms=duration_ms,
-            )
-
-
-class CacheSelfTest(SelfTest):
-    """Test cache (Redis) connectivity."""
-
-    name = "Cache Connection"
-    description = "Verify that the cache system is accessible and functional"
-
-    def run(self) -> SelfTestResult:
-        """Test cache by setting and getting a test value."""
-        start_time = time.time()
-        test_key = "selftest:cache:ping"
-        test_value = "pong"
-
-        try:
-            # Try to set a value
-            cache.set(test_key, test_value, timeout=10)
-
-            # Try to get the value back
-            retrieved_value = cache.get(test_key)
-
-            # Clean up
-            cache.delete(test_key)
-
-            duration_ms = (time.time() - start_time) * 1000
-
-            if retrieved_value == test_value:
-                cache_backend = settings.CACHES.get("default", {}).get(
-                    "BACKEND", "unknown"
-                )
-                return SelfTestResult(
-                    name=self.name,
-                    success=True,
-                    message="Cache connection successful",
-                    details={"backend": cache_backend},
-                    duration_ms=duration_ms,
-                )
-            return SelfTestResult(
-                name=self.name,
-                success=False,
-                message="Cache value mismatch",
-                details={
-                    "expected": test_value,
-                    "received": retrieved_value,
-                },
-                duration_ms=duration_ms,
-            )
-        except (OSError, ValueError, TimeoutError) as e:
-            duration_ms = (time.time() - start_time) * 1000
-            return SelfTestResult(
-                name=self.name,
-                success=False,
-                message=f"Cache connection failed: {str(e)}",
-                details={"exception": str(e)},
-                duration_ms=duration_ms,
-            )
-
-
-class OpenSearchSelfTest(SelfTest):
-    """Test OpenSearch connectivity."""
-
-    name = "OpenSearch Connection"
-    description = "Verify that OpenSearch is accessible and responsive"
-
-    def run(self) -> SelfTestResult:
-        """Test OpenSearch connection by checking cluster health."""
-        start_time = time.time()
-        try:
-            client = opensearch_client()
-
-            # Ping the cluster
-            if not client.ping():
-                duration_ms = (time.time() - start_time) * 1000
-                return SelfTestResult(
-                    name=self.name,
-                    success=False,
-                    message="OpenSearch ping failed",
-                    duration_ms=duration_ms,
-                )
-
-            # Get cluster health
-            health = client.cluster.health()
-
-            duration_ms = (time.time() - start_time) * 1000
-
-            return SelfTestResult(
-                name=self.name,
-                success=True,
-                message="OpenSearch connection successful",
-                details={
-                    "cluster_name": health.get("cluster_name", "unknown"),
-                    "status": health.get("status", "unknown"),
-                    "number_of_nodes": health.get("number_of_nodes", 0),
-                    "number_of_data_nodes": health.get("number_of_data_nodes", 0),
-                    "active_shards": health.get("active_shards", 0),
-                    "host": settings.OPENSEARCH_HOST,
-                    "port": settings.OPENSEARCH_PORT,
-                },
-                duration_ms=duration_ms,
-            )
-        except (OSError, ValueError, TimeoutError) as e:
-            duration_ms = (time.time() - start_time) * 1000
-            return SelfTestResult(
-                name=self.name,
-                success=False,
-                message=f"OpenSearch connection failed: {str(e)}",
-                details={"exception": str(e)},
-                duration_ms=duration_ms,
-            )
-
-
-class SentrySelfTest(SelfTest):
-    """Test Sentry connectivity."""
-
-    name = "Sentry Connection"
-    description = "Verify that Sentry is accessible and responsive"
-
-    def run(self) -> SelfTestResult:
-        """Test Sentry connection by sending a test error."""
-        if not sentry_sdk:
-            return SelfTestResult(
-                name=self.name,
-                success=False,
-                message="Sentry SDK not available",
-            )
-
-        # Check if Sentry DSN is configured
-        sentry_dsn = getattr(settings, "SENTRY_DSN", None)
-        if not sentry_dsn:
-            return SelfTestResult(
-                name=self.name,
-                success=False,
-                message="Sentry DSN not configured",
-            )
-
-        start_time = time.time()
-        try:
-            # Send a test error to Sentry to verify connectivity
-            scope = sentry_sdk.get_current_scope()
-            scope.set_extra("selftest", "Sentry connectivity test")
-
-            try:
-                # Raise a fake error that we'll catch and send to Sentry
-                raise ValueError(
-                    "Sentry self-test error - fake error for connectivity check"
-                )
-            except ValueError as test_error:
-                sentry_sdk.capture_exception(test_error)
-
-            duration_ms = (time.time() - start_time) * 1000
-
-            return SelfTestResult(
-                name=self.name,
-                success=True,
-                message="Sentry connection successful",
-                details={"dsn_configured": bool(sentry_dsn)},
-                duration_ms=duration_ms,
-            )
-        except Exception as e:  # noqa: BLE001 pylint: disable=broad-except
-            duration_ms = (time.time() - start_time) * 1000
-            return SelfTestResult(
-                name=self.name,
-                success=False,
-                message=f"Sentry connection failed: {str(e)}",
-                details={"exception": str(e)},
-                duration_ms=duration_ms,
-            )
-
-
-# Register all built-in tests
-registry.register(DatabaseSelfTest)
-registry.register(CacheSelfTest)
-registry.register(OpenSearchSelfTest)
-
-if settings.SENTRY_DSN:
-    registry.register(SentrySelfTest)

src/backend/core/services/embedding.py

@@ -1,42 +0,0 @@
-"""OpenSearch embedding utilities."""
-
-import logging
-
-from django.conf import settings
-
-import requests
-
-logger = logging.getLogger(__name__)
-
-
-def embed_text(text):
-    """
-    Get embedding vector for the given text from any OpenAI-compatible embedding API
-    """
-    logger.info("embed: '%s'", text)
-
-    response = requests.post(
-        settings.EMBEDDING_API_PATH,
-        headers={"Authorization": f"Bearer {settings.EMBEDDING_API_KEY}"},
-        json={
-            "input": text,
-            "model": settings.EMBEDDING_API_MODEL_NAME,
-            "dimensions": settings.EMBEDDING_DIMENSION,
-            "encoding_format": "float",
-        },
-        timeout=settings.EMBEDDING_REQUEST_TIMEOUT,
-    )
-
-    try:
-        response.raise_for_status()
-    except requests.HTTPError as e:
-        logger.warning("embedding API request failed: %s", str(e))
-        return None
-
-    try:
-        embedding = response.json()["data"][0]["embedding"]
-    except (KeyError, IndexError, TypeError):
-        logger.warning("unexpected embedding response format: %s", response.text)
-        return None
-
-    return embedding

src/backend/core/services/indexing.py

@@ -1,151 +0,0 @@
-"""OpenSearch indexing utilities."""
-
-import logging
-
-from django.conf import settings
-from django.core.exceptions import SuspiciousOperation
-
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from opensearchpy.exceptions import NotFoundError
-from py3langid.langid import MODEL_FILE, LanguageIdentifier
-
-from core.services.opensearch_configuration import (
-    ANALYZERS,
-    FILTERS,
-    MAPPINGS,
-)
-
-from ..models import Service, get_opensearch_index_name
-from .embedding import embed_text
-from .opensearch import check_hybrid_search_enabled, opensearch_client
-
-logger = logging.getLogger(__name__)
-
-
-# see https://pypi.org/project/py3langid/
-LANGUAGE_IDENTIFIER = LanguageIdentifier.from_pickled_model(MODEL_FILE, norm_probs=True)
-LANGUAGE_IDENTIFIER.set_languages(["en", "fr", "de", "nl"])
-
-TEXT_SPLITER = RecursiveCharacterTextSplitter(
-    chunk_size=settings.CHUNK_SIZE,
-    chunk_overlap=settings.CHUNK_OVERLAP,
-)
-
-
-def ensure_index_exists(index_name):
-    """Create index if it does not exist"""
-    try:
-        opensearch_client().indices.get(index=index_name)
-    except NotFoundError:
-        logger.info("Creating index: %s", index_name)
-        opensearch_client().indices.create(
-            index=index_name,
-            body={
-                "settings": {
-                    "index.knn": True,
-                    "analysis": {
-                        "analyzer": ANALYZERS,
-                        "filter": FILTERS,
-                    },
-                },
-                "mappings": MAPPINGS,
-            },
-        )
-
-
-def prepare_document_for_indexing(document):
-    """Prepare document for indexing using nested language structure and handle embedding"""
-    language_code = detect_language_code(f"{document['title']} {document['content']}")
-    return {
-        "id": document["id"],
-        f"title.{language_code}": document["title"],
-        f"content.{language_code}": document["content"],
-        "embedding_model": settings.EMBEDDING_API_MODEL_NAME
-        if check_hybrid_search_enabled()
-        else None,
-        "chunks": chunk_document(
-            document["title"],
-            document["content"],
-        )
-        if check_hybrid_search_enabled()
-        else None,
-        "depth": document["depth"],
-        "path": document["path"],
-        "numchild": document["numchild"],
-        "created_at": document["created_at"],
-        "updated_at": document["updated_at"],
-        "size": document["size"],
-        "users": document["users"],
-        "groups": document["groups"],
-        "reach": document["reach"],
-        "tags": document.get("tags", []),
-        "is_active": document["is_active"],
-    }
-
-
-def chunk_document(title, content):
-    """
-    Chunk a document into multiple pieces and embed them.
-    """
-    chunks = []
-    for idx, chunked_content in enumerate(TEXT_SPLITER.split_text(content)):
-        embedding = embed_text(format_document(title, chunked_content))
-
-        if not embedding:
-            logger.warning(
-                "Failed to embed chunk %d of document '%s'. Document embedding is skipped",
-                idx,
-                title,
-            )
-            # if embedding fails for any chunk, we skip chunking the document
-            return None
-
-        chunks.append(
-            {
-                "index": idx,
-                "content": chunked_content,
-                "embedding": embedding,
-            }
-        )
-
-    logger.info("Document %s chunked into %d pieces", title, len(chunks))
-    return chunks
-
-
-def format_document(title, content):
-    """Get the embedding input format for a document"""
-    return f"<{title}>:<{content}>"
-
-
-def detect_language_code(text):
-    """Detect the language code of the document content."""
-
-    detected_code, confidence = LANGUAGE_IDENTIFIER.classify(text)
-
-    if confidence < settings.LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD:
-        return settings.UNDETERMINED_LANGUAGE_CODE
-
-    return detected_code
-
-
-def get_opensearch_indices(audience, services):
-    """
-    Get OpenSearch indices for the given audience and services.
-    """
-    try:
-        user_service = Service.objects.get(client_id=audience, is_active=True)
-    except Service.DoesNotExist as e:
-        logger.warning("Login failed: No service %s found", audience)
-        raise SuspiciousOperation("Service is not available") from e
-
-    # Find allowed sub-services for this service
-    allowed_services = set(user_service.services.values_list("name", flat=True))
-    allowed_services.add(user_service.name)
-
-    if services:
-        available_service = set(services).intersection(allowed_services)
-
-        if len(available_service) < len(services):
-            raise SuspiciousOperation("Some requested services are not available")
-
-    return [get_opensearch_index_name(service) for service in allowed_services]

src/backend/core/services/opensearch.py

@@ -1,66 +0,0 @@
-"""OpenSearch common utilities."""
-
-import logging
-from functools import cache
-
-from django.conf import settings
-
-from opensearchpy import OpenSearch
-from rest_framework.exceptions import ValidationError
-
-logger = logging.getLogger(__name__)
-
-
-REQUIRED_ENV_VARIABLES = [
-    "OPENSEARCH_HOST",
-    "OPENSEARCH_PORT",
-    "OPENSEARCH_USER",
-    "OPENSEARCH_PASSWORD",
-    "OPENSEARCH_USE_SSL",
-]
-
-
-@cache
-def opensearch_client():
-    """Get OpenSearch client, ensuring required env variables are set"""
-    missing_env_variables = [
-        variable
-        for variable in REQUIRED_ENV_VARIABLES
-        if getattr(settings, variable, None) is None
-    ]
-    if missing_env_variables:
-        raise ValidationError(
-            f"Missing required OpenSearch environment variables: {', '.join(missing_env_variables)}"
-        )
-
-    return OpenSearch(
-        hosts=[{"host": settings.OPENSEARCH_HOST, "port": settings.OPENSEARCH_PORT}],
-        http_auth=(settings.OPENSEARCH_USER, settings.OPENSEARCH_PASSWORD),
-        timeout=50,
-        use_ssl=settings.OPENSEARCH_USE_SSL,
-        verify_certs=False,
-    )
-
-
-@cache
-def check_hybrid_search_enabled():
-    """Check that all required environment variables are set for hybrid search."""
-    if settings.HYBRID_SEARCH_ENABLED is not True:
-        logger.info("Hybrid search is disabled via HYBRID_SEARCH_ENABLED setting")
-        return False
-
-    required_vars = [
-        "HYBRID_SEARCH_WEIGHTS",
-        "EMBEDDING_API_PATH",
-        "EMBEDDING_API_KEY",
-        "EMBEDDING_API_MODEL_NAME",
-        "EMBEDDING_DIMENSION",
-    ]
-    missing_vars = [var for var in required_vars if not getattr(settings, var, None)]
-    if missing_vars:
-        logger.warning(
-            "Missing variables for hybrid search: %s", ", ".join(missing_vars)
-        )
-        return False
-
-    return True

src/backend/core/services/opensearch_configuration.py

@@ -1,292 +0,0 @@
-"""OpenSearch configuration."""
-
-from django.conf import settings
-
-ANALYZERS = {
-    "french_analyzer": {
-        "type": "custom",
-        "tokenizer": "standard",
-        "filter": [
-            "lowercase",
-            "asciifolding",
-            "french_elision",
-            "french_stop",
-            "french_stemmer",
-        ],
-    },
-    "english_analyzer": {
-        "type": "custom",
-        "tokenizer": "standard",
-        "filter": [
-            "lowercase",
-            "asciifolding",
-            "english_stop",
-            "english_stemmer",
-        ],
-    },
-    "german_analyzer": {
-        "type": "custom",
-        "tokenizer": "standard",
-        "filter": [
-            "lowercase",
-            "asciifolding",
-            "german_stop",
-            "german_stemmer",
-        ],
-    },
-    "dutch_analyzer": {
-        "type": "custom",
-        "tokenizer": "standard",
-        "filter": [
-            "lowercase",
-            "asciifolding",
-            "dutch_stop",
-            "dutch_stemmer",
-        ],
-    },
-    "undetermined_language_analyzer": {
-        "type": "custom",
-        "tokenizer": "standard",
-        "filter": [
-            "lowercase",
-            "asciifolding",
-        ],
-    },
-    "trigram_analyzer": {
-        "type": "custom",
-        "tokenizer": "standard",
-        "filter": [
-            "lowercase",
-            "asciifolding",
-            "trigram_filter",
-        ],
-    },
-}
-
-FILTERS = {
-    "french_elision": {
-        "type": "elision",
-        "articles_case": True,
-        "articles": [
-            "l",
-            "m",
-            "t",
-            "qu",
-            "n",
-            "s",
-            "j",
-            "d",
-            "c",
-            "jusqu",
-            "quoiqu",
-            "lorsqu",
-            "puisqu",
-        ],
-    },
-    "french_stop": {
-        "type": "stop",
-        "stopwords": "_french_",
-    },
-    "french_stemmer": {
-        "type": "stemmer",
-        "language": "light_french",
-    },
-    "english_stop": {
-        "type": "stop",
-        "stopwords": "_english_",
-    },
-    "english_stemmer": {
-        "type": "stemmer",
-        "language": "english",
-    },
-    "german_stop": {
-        "type": "stop",
-        "stopwords": "_german_",
-    },
-    "german_stemmer": {
-        "type": "stemmer",
-        "language": "light_german",
-    },
-    "dutch_stop": {
-        "type": "stop",
-        "stopwords": "_dutch_",
-    },
-    "dutch_stemmer": {
-        "type": "stemmer",
-        "language": "dutch",
-    },
-    "trigram_filter": {
-        "type": "ngram",
-        "min_gram": 3,
-        "max_gram": 3,
-    },
-}
-
-MAPPINGS = {
-    "dynamic": "strict",
-    "properties": {
-        "id": {"type": "keyword"},
-        # French
-        "title.fr": {
-            "type": "keyword",
-            "fields": {
-                "text": {
-                    "type": "text",
-                    "analyzer": "french_analyzer",
-                    "fields": {
-                        "trigrams": {
-                            "type": "text",
-                            "analyzer": "trigram_analyzer",
-                        }
-                    },
-                }
-            },
-        },
-        "content.fr": {
-            "type": "text",
-            "analyzer": "french_analyzer",
-            "fields": {
-                "trigrams": {
-                    "type": "text",
-                    "analyzer": "trigram_analyzer",
-                }
-            },
-        },
-        # English
-        "title.en": {
-            "type": "keyword",
-            "fields": {
-                "text": {
-                    "type": "text",
-                    "analyzer": "english_analyzer",
-                    "fields": {
-                        "trigrams": {
-                            "type": "text",
-                            "analyzer": "trigram_analyzer",
-                        }
-                    },
-                }
-            },
-        },
-        "content.en": {
-            "type": "text",
-            "analyzer": "english_analyzer",
-            "fields": {
-                "trigrams": {
-                    "type": "text",
-                    "analyzer": "trigram_analyzer",
-                }
-            },
-        },
-        # German
-        "title.de": {
-            "type": "keyword",
-            "fields": {
-                "text": {
-                    "type": "text",
-                    "analyzer": "german_analyzer",
-                    "fields": {
-                        "trigrams": {
-                            "type": "text",
-                            "analyzer": "trigram_analyzer",
-                        }
-                    },
-                }
-            },
-        },
-        "content.de": {
-            "type": "text",
-            "analyzer": "german_analyzer",
-            "fields": {
-                "trigrams": {
-                    "type": "text",
-                    "analyzer": "trigram_analyzer",
-                }
-            },
-        },
-        # Dutch
-        "title.nl": {
-            "type": "keyword",
-            "fields": {
-                "text": {
-                    "type": "text",
-                    "analyzer": "dutch_analyzer",
-                    "fields": {
-                        "trigrams": {
-                            "type": "text",
-                            "analyzer": "trigram_analyzer",
-                        }
-                    },
-                }
-            },
-        },
-        "content.nl": {
-            "type": "text",
-            "analyzer": "dutch_analyzer",
-            "fields": {
-                "trigrams": {
-                    "type": "text",
-                    "analyzer": "trigram_analyzer",
-                }
-            },
-        },
-        # Undetermined language
-        "title.und": {
-            "type": "keyword",
-            "fields": {
-                "text": {
-                    "type": "text",
-                    "analyzer": "undetermined_language_analyzer",
-                    "fields": {
-                        "trigrams": {
-                            "type": "text",
-                            "analyzer": "trigram_analyzer",
-                        }
-                    },
-                }
-            },
-        },
-        "content.und": {
-            "type": "text",
-            "analyzer": "undetermined_language_analyzer",
-            "fields": {
-                "trigrams": {
-                    "type": "text",
-                    "analyzer": "trigram_analyzer",
-                }
-            },
-        },
-        "depth": {"type": "integer"},
-        "path": {
-            "type": "keyword",
-            "fields": {"text": {"type": "text"}},
-        },
-        "numchild": {"type": "integer"},
-        "created_at": {"type": "date"},
-        "updated_at": {"type": "date"},
-        "size": {"type": "long"},
-        "users": {"type": "keyword"},
-        "groups": {"type": "keyword"},
-        "reach": {"type": "keyword"},
-        "tags": {"type": "keyword"},
-        "is_active": {"type": "boolean"},
-        "chunks": {
-            "type": "nested",
-            "properties": {
-                "index": {"type": "integer"},
-                "content": {"type": "text"},
-                "embedding": {
-                    "type": "knn_vector",
-                    "dimension": settings.EMBEDDING_DIMENSION,
-                    "method": {
-                        "engine": "lucene",
-                        "space_type": "l2",
-                        "name": "hnsw",
-                        "parameters": {},
-                    },
-                },
-            },
-        },
-        "embedding_model": {"type": "keyword"},
-    },
-}

src/backend/core/services/search.py

@@ -1,247 +0,0 @@
-"""OpenSearch search utilities."""
-
-import logging
-
-from django.conf import settings
-
-from core import enums
-from core.enums import SearchTypeEnum
-
-from .embedding import embed_text
-from .opensearch import check_hybrid_search_enabled, opensearch_client
-
-logger = logging.getLogger(__name__)
-
-
-# pylint: disable=too-many-arguments, too-many-positional-arguments
-def search(  # noqa : PLR0913
-    q,
-    nb_results,
-    order_by,
-    order_direction,
-    search_indices,
-    reach,
-    visited,
-    user_sub,
-    groups,
-    tags,
-    search_type,
-    path=None,
-):
-    """Perform an OpenSearch search"""
-    query = get_query(
-        q=q,
-        nb_results=nb_results,
-        reach=reach,
-        visited=visited,
-        user_sub=user_sub,
-        groups=groups,
-        tags=tags,
-        path=path,
-        search_type=search_type,
-    )
-    return opensearch_client().search(  # pylint: disable=unexpected-keyword-arg
-        index=",".join(search_indices),
-        body={
-            "_source": enums.SOURCE_FIELDS,  # limit the fields to return
-            "script_fields": {
-                "number_of_users": {"script": {"source": "doc['users'].size()"}},
-                "number_of_groups": {"script": {"source": "doc['groups'].size()"}},
-            },
-            "sort": get_sort(
-                query_keys=query.keys(),
-                order_by=order_by,
-                order_direction=order_direction,
-            ),
-            "size": nb_results,
-            "query": query,
-        },
-        params=get_params(query_keys=query.keys()),
-        # disable=unexpected-keyword-arg because
-        # ignore_unavailable is not in the method declaration
-        ignore_unavailable=True,
-    )
-
-
-# pylint: disable=too-many-arguments, too-many-positional-arguments
-def get_query(  # noqa : PLR0913
-    q,
-    nb_results,
-    reach,
-    visited,
-    user_sub,
-    groups,
-    tags,
-    search_type,
-    path=None,
-):
-    """Build OpenSearch query body based on parameters"""
-    filter_ = get_filter(reach, visited, user_sub, groups, tags, path)
-
-    if q == "*":
-        logger.info("Performing match_all query")
-        return {
-            "bool": {
-                "must": {"match_all": {}},
-                "filter": {"bool": {"filter": filter_}},
-            },
-        }
-
-    q_vector = vectorize_query(q, search_type)
-
-    if not q_vector:
-        logger.info("Performing full-text search without embedding: %s", q)
-        return get_full_text_query(q, filter_)
-
-    logger.info("Performing hybrid search with embedding: %s", q)
-    return {
-        "hybrid": {
-            "queries": [
-                get_full_text_query(q, filter_),
-                get_semantic_search_query(q_vector, filter_, nb_results),
-            ],
-        }
-    }
-
-
-def vectorize_query(q, search_type):
-    """Vectorize the query if hybrid search is enabled and requested"""
-    hybrid_search_enabled = check_hybrid_search_enabled()
-
-    if search_type == SearchTypeEnum.HYBRID:
-        if not hybrid_search_enabled:
-            logger.warning(
-                "Hybrid search was requested (search_type=hybrid) but is disabled on server",
-            )
-            return None
-
-        return embed_text(q)
-
-    return None
-
-
-def get_semantic_search_query(q_vector, filter_, nb_results):
-    """Build OpenSearch semantic search query"""
-    return {
-        "bool": {
-            "must": {
-                "nested": {
-                    "path": "chunks",
-                    "score_mode": "max",
-                    "query": {
-                        "knn": {
-                            "chunks.embedding": {
-                                "vector": q_vector,
-                                "k": nb_results,
-                            }
-                        }
-                    },
-                }
-            },
-            "filter": filter_,
-        }
-    }
-
-
-def get_full_text_query(q, filter_):
-    """Build OpenSearch full-text query"""
-    return {
-        "bool": {
-            "must": {
-                "bool": {
-                    "should": [
-                        {
-                            "multi_match": {
-                                "query": q,
-                                "fields": [
-                                    "title.*.text^3",
-                                    "content.*",
-                                ],
-                            }
-                        },
-                        {
-                            "multi_match": {
-                                "query": q,
-                                "fields": [
-                                    "title.*.text.trigrams^3",
-                                    "content.*.trigrams",
-                                ],
-                                "boost": settings.TRIGRAMS_BOOST,
-                                "minimum_should_match": settings.TRIGRAMS_MINIMUM_SHOULD_MATCH,
-                            }
-                        },
-                    ],
-                    "minimum_should_match": 1,
-                },
-            },
-            "filter": filter_,
-        }
-    }
-
-
-# pylint: disable=too-many-arguments, too-many-positional-arguments
-def get_filter(  # noqa : PLR0913
-    reach, visited, user_sub, groups, tags, path=None
-):
-    """Build OpenSearch filter"""
-    filters = [
-        {"term": {"is_active": True}},  # filter out inactive documents
-        # Access control filters
-        {
-            "bool": {
-                "should": [
-                    # Public or authenticated (not restricted)
-                    {
-                        "bool": {
-                            "must_not": {
-                                "term": {enums.REACH: enums.ReachEnum.RESTRICTED},
-                            },
-                            "must": {
-                                "terms": {"_id": sorted(visited)},
-                            },
-                        }
-                    },
-                    # Restricted: either user or group must match
-                    {"term": {enums.USERS: user_sub}},
-                    {"terms": {enums.GROUPS: groups}},
-                ],
-                "minimum_should_match": 1,
-            }
-        },
-    ]
-
-    # Optional reach filter
-    if reach is not None:
-        filters.append({"term": {enums.REACH: reach}})
-
-    # Optional tags filter
-    if tags:
-        # logical or: if tags are provided the matching documents should have at least one of them
-        filters.append({"terms": {"tags": tags}})
-
-    # Optional path filter
-    if path:
-        # filter documents that start with the provided path
-        filters.append({"prefix": {"path": path}})
-
-    return filters
-
-
-def get_sort(query_keys, order_by, order_direction):
-    """Build OpenSearch sort clause"""
-    # Add sorting logic based on relevance or specified field
-    if "hybrid" in query_keys:
-        # sorting by other field than "_score" is not supported in hybrid search
-        # see: https://github.com/opensearch-project/neural-search/issues/866
-        return {"_score": {"order": order_direction}}
-    if order_by == enums.RELEVANCE:
-        return {"_score": {"order": order_direction}}
-
-    return {order_by: {"order": order_direction}}
-
-
-def get_params(query_keys):
-    """Build OpenSearch search parameters"""
-    if "hybrid" in query_keys:
-        return {"search_pipeline": settings.HYBRID_SEARCH_PIPELINE_ID}
-    return {}

src/backend/core/templates/admin/core/services/change_list.html

@@ -1,13 +0,0 @@
-{% extends "admin/change_list.html" %}
-
-{% load i18n admin_urls %}
-
-{% block object-tools-items %}
-  <li>
-    <a href="{% url 'admin:core_service_ensure_search_pipeline' %}">
-      {% trans "Ensure search pipeline" %}
-    </a>
-  </li>
-  {{ block.super }}
-{% endblock %}
-

src/backend/core/templates/admin/index.html

@@ -1,26 +0,0 @@
-{% extends "admin/index.html" %}
-{% load i18n %}
-
-{% block sidebar %}
-{{ block.super }}
-
-<div id="content-main">
-    <div class="module" style="margin-top: 20px;">
-        <table>
-            <caption>
-                {% trans 'System Tools' %}
-            </caption>
-            <tbody>
-                <tr>
-                    <th scope="row">
-                        <a href="{% url 'admin:selftest' %}">{% trans 'System Self-Tests' %}</a>
-                    </th>
-                    <td>{% trans 'Run selftest checks' %}</td>
-                </tr>
-            </tbody>
-        </table>
-    </div>
-</div>
-
-{% endblock %}
-

src/backend/core/templates/admin/selftest.html

@@ -1,258 +0,0 @@
-{% extends "admin/base_site.html" %}
-{% load i18n static %}
-
-{% block extrastyle %}
-    {{ block.super }}
-    <style>
-        .selftest-container {
-            margin: 20px;
-        }
-
-        .selftest-module {
-            background: var(--body-bg);
-            border: 1px solid var(--border-color, #ddd);
-            border-radius: 4px;
-            margin-bottom: 20px;
-        }
-
-        .selftest-module h2 {
-            background: var(--primary);
-            color: var(--primary-fg);
-            padding: 10px 15px;
-            margin: 0;
-            font-size: 14px;
-            font-weight: normal;
-        }
-
-        .module-content {
-            padding: 15px;
-        }
-
-        .run-tests-btn {
-            display: inline-block;
-            padding: 10px 20px;
-            background: var(--primary);
-            color: var(--primary-fg);
-            text-decoration: none;
-            border-radius: 4px;
-            font-weight: bold;
-            margin-bottom: 20px;
-            border: none;
-            cursor: pointer;
-        }
-
-        .run-tests-btn:hover {
-            background: var(--primary-hover, #205067);
-        }
-
-        .overall-status {
-            padding: 15px;
-            border-radius: 4px;
-            font-weight: bold;
-            text-align: center;
-        }
-
-        .overall-status.success {
-            background-color: var(--success-bg, #d4edda);
-            color: var(--success-fg, #155724);
-            border: 2px solid var(--success-border, #c3e6cb);
-        }
-
-        .overall-status.failure {
-            background-color: var(--error-bg, #f8d7da);
-            color: var(--error-fg, #721c24);
-            border: 2px solid var(--error-border, #f5c6cb);
-        }
-
-        .test-item {
-            border: 1px solid var(--border-color, #ddd);
-            border-radius: 4px;
-            margin-bottom: 15px;
-            overflow: hidden;
-            background: var(--body-bg);
-        }
-
-        .test-header {
-            padding: 12px 15px;
-            background-color: var(--darkened-bg);
-            display: flex;
-            justify-content: space-between;
-            align-items: center;
-            border-bottom: 1px solid var(--border-color, #ddd);
-        }
-
-        .test-name {
-            font-weight: bold;
-            font-size: 14px;
-            color: var(--body-fg);
-        }
-
-        .test-status {
-            padding: 4px 12px;
-            border-radius: 3px;
-            font-size: 11px;
-            font-weight: bold;
-            text-transform: uppercase;
-        }
-
-        .test-status.success {
-            background-color: #28a745;
-            color: white;
-        }
-
-        .test-status.failure {
-            background-color: #dc3545;
-            color: white;
-        }
-
-        .test-body {
-            padding: 15px;
-            background-color: var(--body-bg);
-        }
-
-        .test-message {
-            margin-bottom: 10px;
-            color: var(--body-fg);
-        }
-
-        .test-details {
-            background-color: var(--darkened-bg);
-            padding: 10px;
-            border-radius: 4px;
-            font-family: "Courier New", Monaco, monospace;
-            font-size: 12px;
-            border: 1px solid var(--border-color, #ddd);
-        }
-
-        .test-details dl {
-            margin: 0;
-        }
-
-        .test-details dt {
-            font-weight: bold;
-            color: var(--body-fg);
-            margin-top: 5px;
-        }
-
-        .test-details dd {
-            margin-left: 20px;
-            color: var(--body-quiet-color);
-            margin-bottom: 5px;
-        }
-
-        .duration {
-            color: var(--body-quiet-color);
-            font-size: 11px;
-            font-style: italic;
-        }
-
-        .available-tests-list {
-            list-style: none;
-            padding: 0;
-            margin: 0;
-        }
-
-        .available-tests-list li {
-            padding: 10px;
-            border-bottom: 1px solid var(--hairline-color, #eee);
-            list-style-type: none;
-        }
-
-        .available-tests-list li:last-child {
-            border-bottom: none;
-        }
-
-        .test-description {
-            color: var(--body-quiet-color);
-            font-size: 13px;
-            display: block;
-            margin-top: 4px;
-        }
-    </style>
-{% endblock %}
-
-{% block breadcrumbs %}
-<div class="breadcrumbs">
-    <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
-    &rsaquo; {% trans 'System Self-Tests' %}
-</div>
-{% endblock %}
-
-{% block content %}
-<div class="selftest-container">
-    {% if not run_tests %}
-        <a href="?run=true" class="run-tests-btn">{% trans "Run All Tests" %}</a>
-    {% else %}
-        <a href="{% url 'admin:selftest' %}" class="run-tests-btn" style="opacity: 0.8;">{% trans "Reset" %}</a>
-    {% endif %}
-
-    {% if run_tests %}
-        {% if all_passed is not None %}
-        <div class="selftest-module">
-            <div class="overall-status {% if all_passed %}success{% else %}failure{% endif %}">
-                {% if all_passed %}
-                    ✅ {% trans "All tests passed successfully!" %}
-                {% else %}
-                    ❌ {% trans "Some tests failed. Please check the details below." %}
-                {% endif %}
-            </div>
-        </div>
-        {% endif %}
-
-        <div class="selftest-module">
-            <h2>{% trans "Test Results" %}</h2>
-            <div class="module-content">
-                {% for result in results %}
-                    <div class="test-item">
-                        <div class="test-header">
-                            <span class="test-name">{{ result.name }}</span>
-                            <span class="test-status {% if result.success %}success{% else %}failure{% endif %}">
-                                {% if result.success %}
-                                    ✓ {% trans "Pass" %}
-                                {% else %}
-                                    ✗ {% trans "Fail" %}
-                                {% endif %}
-                            </span>
-                        </div>
-                        <div class="test-body">
-                            <div class="test-message">
-                                {{ result.message }}
-                                {% if result.duration_ms %}
-                                    <span class="duration">({{ result.duration_ms|floatformat:2 }}ms)</span>
-                                {% endif %}
-                            </div>
-                            {% if result.details %}
-                                <div class="test-details">
-                                    <strong>{% trans "Details:" %}</strong>
-                                    <dl>
-                                        {% for key, value in result.details.items %}
-                                            <dt>{{ key }}:</dt>
-                                            <dd>{{ value }}</dd>
-                                        {% endfor %}
-                                    </dl>
-                                </div>
-                            {% endif %}
-                        </div>
-                    </div>
-                {% endfor %}
-            </div>
-        </div>
-    {% else %}
-        <div class="selftest-module">
-            <h2>{% trans "Available Tests" %}</h2>
-            <div class="module-content">
-                <p>{% trans "Click 'Run All Tests' to execute the following system health checks:" %}</p>
-                <ul class="available-tests-list">
-                    {% for test in available_tests %}
-                        <li>
-                            <strong>{{ test.name }}</strong>
-                            <span class="test-description">{{ test.description }}</span>
-                        </li>
-                    {% endfor %}
-                </ul>
-            </div>
-        </div>
-    {% endif %}
-</div>
-{% endblock %}
-

src/backend/core/tests/commands/test_create_search_pipeline.py

@@ -1,76 +0,0 @@
-"""
-Unit test for `create_search_pipeline` command.
-"""
-
-import logging
-
-from django.core.management import call_command
-
-import pytest
-
-from core.services.opensearch import opensearch_client
-from core.tests.utils import (
-    enable_hybrid_search,
-)
-from core.utils import delete_search_pipeline
-
-
-@pytest.fixture(autouse=True)
-def before_each():
-    """Delete search pipeline before each test"""
-    delete_search_pipeline()
-    yield
-    delete_search_pipeline()
-
-
-def test_create_search_pipeline(settings, caplog):
-    """Test command create search pipeline"""
-
-    enable_hybrid_search(settings)
-
-    with caplog.at_level(logging.INFO):
-        call_command("create_search_pipeline")
-
-    assert any(
-        f"Creating search pipeline: {settings.HYBRID_SEARCH_PIPELINE_ID}" in message
-        for message in caplog.messages
-    )
-
-    # calling get works without raising NotFoundError
-    opensearch_client().search_pipeline.get(id=settings.HYBRID_SEARCH_PIPELINE_ID)
-
-
-def test_create_search_pipeline_but_it_exists_already(settings, caplog):
-    """Test command create search pipeline but it already exists"""
-
-    opensearch_client().transport.perform_request(
-        method="PUT",
-        url="/_search/pipeline/" + settings.HYBRID_SEARCH_PIPELINE_ID,
-        body={
-            "description": "Post processor for hybrid search",
-            "phase_results_processors": [
-                {
-                    "normalization-processor": {
-                        "combination": {
-                            "technique": "arithmetic_mean",
-                            "parameters": {"weights": settings.HYBRID_SEARCH_WEIGHTS},
-                        }
-                    }
-                }
-            ],
-        },
-    )
-
-    with caplog.at_level(logging.INFO):
-        call_command("create_search_pipeline")
-
-    assert any(
-        "Search pipeline exists already" in message for message in caplog.messages
-    )
-    assert not any(
-        f"Creating search pipeline: {settings.HYBRID_SEARCH_PIPELINE_ID}" in message
-        for message in caplog.messages
-    )
-
-    # the pipeline is still here
-    opensearch_client().search_pipeline.get(id=settings.HYBRID_SEARCH_PIPELINE_ID)

src/backend/core/tests/commands/test_reindex_with_embedding.py

@@ -1,291 +0,0 @@
-"""
-Unit test for `reindex_with_embedding` command.
-"""
-
-from unittest.mock import patch
-
-from django.core.management import CommandError, call_command
-
-import pytest
-import responses
-
-from core.management.commands.reindex_with_embedding import (
-    check_hybrid_search_enabled as check_hybrid_search_enabled_command,
-)
-from core.management.commands.reindex_with_embedding import (
-    reindex_with_embedding,
-)
-from core.models import get_opensearch_index_name
-from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
-from core.tests.mock import albert_embedding_response
-from core.tests.utils import (
-    enable_hybrid_search,
-)
-from core.utils import (
-    bulk_create_documents,
-    delete_search_pipeline,
-    get_language_value,
-    prepare_index,
-)
-
-SERVICE_NAME = "test-index"
-
-
-@pytest.fixture(autouse=True)
-def before_each():
-    """Clear caches and delete search pipeline before each test"""
-    clear_caches()
-    yield
-    clear_caches()
-
-
-def clear_caches():
-    """Clear caches used in opensearch service and factories"""
-    check_hybrid_search_enabled.cache_clear()
-    # the instance of check_hybrid_search_enabled used in utils.py
-    # is different and must be cleared separately
-    check_hybrid_search_enabled_command.cache_clear()
-    delete_search_pipeline()
-
-
-@responses.activate
-def test_reindex_with_embedding_command(settings):
-    """Test command create indexes with embedding and search pipeline"""
-    # create documents and index them with hybrid search disabled
-    opensearch_client_ = opensearch_client()
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    index_name = get_opensearch_index_name(SERVICE_NAME)
-    prepare_index(index_name, documents)
-
-    # the index has not been embedded in the initial state
-    initial_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
-        index=index_name, size=3, body={"query": {"match_all": {}}}
-    )
-    assert len(initial_index["hits"]["hits"]) == 3
-    for embedded_hit in initial_index["hits"]["hits"]:
-        assert embedded_hit["_source"]["chunks"] is None
-        assert embedded_hit["_source"]["embedding_model"] is None
-
-    # enable hybrid search
-    enable_hybrid_search(settings)
-    check_hybrid_search_enabled_command.cache_clear()
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    call_command("reindex_with_embedding", SERVICE_NAME)
-
-    opensearch_client_.indices.refresh(index=index_name)
-    embedded_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
-        index=index_name, size=3, body={"query": {"match_all": {}}}
-    )
-
-    # the source index has been replaced with embedding version
-    assert len(embedded_index["hits"]["hits"]) == 3
-    for embedded_hit in embedded_index["hits"]["hits"]:
-        embedded_source = embedded_hit["_source"]
-        # the index contains a embedding and embedding_model
-        assert len(embedded_source["chunks"]) == 1
-        assert (
-            embedded_source["chunks"][0]["embedding"]
-            == albert_embedding_response.response["data"][0]["embedding"]
-        )
-        assert embedded_source["embedding_model"] == settings.EMBEDDING_API_MODEL_NAME
-        # assert initial value have not been effected
-        initial_hits = [
-            hit_
-            for hit_ in initial_index["hits"]["hits"]
-            if hit_["_id"] == embedded_hit["_id"]
-        ]
-        assert len(initial_hits) == 1
-        initial_source = initial_hits[0]["_source"]
-        assert initial_source["title.en"] == embedded_source["title.en"]
-        assert initial_source["content.en"] == embedded_source["content.en"]
-        assert initial_source["created_at"] == embedded_source["created_at"]
-        assert initial_source["users"] == embedded_source["users"]
-
-
-@responses.activate
-def test_reindex_can_fail_and_restart(settings):
-    """Test command handles embedding errors gracefully and continues processing."""
-    opensearch_client_ = opensearch_client()
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    index_name = get_opensearch_index_name(SERVICE_NAME)
-    prepare_index(index_name, documents)
-
-    # enable hybrid search after first indexing
-    enable_hybrid_search(settings)
-    check_hybrid_search_enabled_command.cache_clear()
-
-    # First call succeeds, second fails, third succeeds
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json={"error": "rate limit exceeded"},
-        status=429,
-    )
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    result = reindex_with_embedding(index_name)
-
-    # assert results reflect 2 successes and 1 failure
-    assert result["nb_success_embedding"] == 2
-    assert result["nb_failed_embedding"] == 1
-
-    # assert the index state
-    opensearch_client_.indices.refresh(index=index_name)
-    embedded_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
-        index=index_name, size=3, body={"query": {"match_all": {}}}
-    )
-    # Should have 2 documents with embeddings, 1 without due to error
-    embedded_count = 0
-    not_embedded_count = 0
-    for hit in embedded_index["hits"]["hits"]:
-        if hit["_source"].get("chunks"):
-            embedded_count += 1
-            assert (
-                hit["_source"]["embedding_model"] == settings.EMBEDDING_API_MODEL_NAME
-            )
-        else:
-            not_embedded_count += 1
-            assert hit["_source"]["embedding_model"] is None
-    assert embedded_count == 2
-    assert not_embedded_count == 1
-
-    # the command can be run again to index failed items
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    result = reindex_with_embedding(index_name)
-
-    # assert results
-    assert result["nb_success_embedding"] == 1
-    assert result["nb_failed_embedding"] == 0
-
-    # assert there is now 1 more success and 0 failures
-    opensearch_client_.indices.refresh(index=index_name)
-    embedded_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
-        index=index_name, size=3, body={"query": {"match_all": {}}}
-    )
-    for hit in embedded_index["hits"]["hits"]:
-        for chunk in hit["_source"]["chunks"]:
-            assert (
-                chunk["embedding"]
-                == albert_embedding_response.response["data"][0]["embedding"]
-            )
-        assert hit["_source"]["embedding_model"] == settings.EMBEDDING_API_MODEL_NAME
-
-
-@responses.activate
-def test_reindex_preserves_concurrent_updates(settings):
-    """
-    Test that concurrent document updates don't get overwritten by reindexing.
-    This test simulates the following scenario:
-    • the hybrid search is disabled
-    • documents are created and indexed without indexing
-    • the hybrid search is enabled
-    • the reindexing is triggered
-    • one document is updated while the reindexing is still running
-    Because the updated document is modified after the hybrid search is enabled,
-    it has properly been indexed with embedding, the reindexing command must
-    ignore this document to preserve this latest update.
-    """
-    opensearch_client_ = opensearch_client()
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-        ]
-    )
-    index_name = get_opensearch_index_name(SERVICE_NAME)
-    prepare_index(index_name, documents)
-    enable_hybrid_search(settings)
-
-    updated_title = "updated dog"
-    patch(
-        "core.services.opensearch.opensearch_client_.search",
-        side_effect=opensearch_client_.update(
-            index=index_name,
-            id=documents[1]["id"],
-            body={
-                "doc": {
-                    "title.en": updated_title,
-                }
-            },
-        ),
-    )
-
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    result = reindex_with_embedding(index_name)
-    assert result["nb_success_embedding"] == 2
-    assert result["nb_failed_embedding"] == 0
-
-    opensearch_client_.indices.refresh(index=index_name)
-    embedded_index = opensearch_client_.search(  # pylint: disable=unexpected-keyword-arg
-        index=index_name, size=2, body={"query": {"match_all": {}}}
-    )
-    # Check that the latest update is preserved
-    updated_document = [
-        hit
-        for hit in embedded_index["hits"]["hits"]
-        if get_language_value(hit["_source"], "title") == updated_title
-    ]
-    assert len(updated_document) == 1
-    # Check it was not embedded
-    assert updated_document[0]["_source"]["chunks"] is None
-    assert updated_document[0]["_source"]["embedding_model"] is None
-
-
-def test_reindex_command_but_hybrid_search_is_disabled():
-    """Test the `reindex_with_embedding` command fails when hybrid search is disabled."""
-    with pytest.raises(CommandError) as err:
-        call_command("reindex_with_embedding", SERVICE_NAME)
-
-    assert str(err.value) == "Hybrid search is not enabled or properly configured."
-
-
-def test_reindex_command_but_index_does_not_exist(settings):
-    """Test the `reindex_with_embedding` command fails when the index does not exist."""
-    wrong_index = "wrong-index-name"
-    enable_hybrid_search(settings)
-
-    wrong_index_name = get_opensearch_index_name(wrong_index)
-
-    with pytest.raises(CommandError) as err:
-        call_command("reindex_with_embedding", wrong_index)
-
-    assert str(err.value) == f"Index {wrong_index_name} does not exist."

src/backend/core/tests/conftest.py

@@ -1,15 +1,9 @@
 """Fixtures for tests in the find core application"""
 
 import pytest
-from faker import Faker
 from lasuite.oidc_resource_server.authentication import (
     get_resource_server_backend,
 )
-from opensearchpy.exceptions import NotFoundError
-
-from core.services import opensearch
-
-fake = Faker()
 
 
 @pytest.fixture(name="jwt_rs_backend")
@@ -26,27 +20,3 @@ def jwt_resource_server_backend_fixture(settings):
 
     settings.OIDC_RS_BACKEND_CLASS = _original_backend
     get_resource_server_backend.cache_clear()
-
-
-@pytest.fixture(autouse=True)
-def cleanup_test_index(settings):
-    """
-    Fixture to set a randomized prefix for all service indexes within the tests
-    and remove them on tear down.
-    """
-    _original_prefix = settings.OPENSEARCH_INDEX_PREFIX
-    prefix = "".join(fake.random_letters(5)).lower()
-    settings.OPENSEARCH_INDEX_PREFIX = prefix
-
-    # Create client here to prevent "teardown" issues when the opensearch settings are
-    # removed for error tests.
-    client = opensearch.opensearch_client()
-
-    yield
-
-    settings.OPENSEARCH_INDEX_PREFIX = _original_prefix
-
-    try:
-        client.indices.delete(index=f"{prefix}-*")
-    except NotFoundError:
-        pass

src/backend/core/tests/test_admin_selftest.py

@@ -1,181 +0,0 @@
-"""Tests for the admin selftest view."""
-
-from unittest.mock import patch
-
-from django.contrib.auth import get_user_model
-from django.urls import reverse
-
-import pytest
-
-from core.selftests import SelfTestResult
-
-pytestmark = pytest.mark.django_db
-
-User = get_user_model()
-
-
-@pytest.fixture(autouse=True)
-def _override_storage_settings(settings):
-    """Override storage settings for all tests."""
-    settings.STORAGES = {
-        "default": {
-            "BACKEND": "django.core.files.storage.FileSystemStorage",
-        },
-        "staticfiles": {
-            "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
-        },
-    }
-
-
-def test_selftest_requires_authentication(client):
-    """Test that the selftest page requires authentication."""
-    selftest_url = reverse("admin:selftest")
-
-    response = client.get(selftest_url)
-
-    # Should redirect to login
-    assert response.status_code == 302
-    assert "/admin/login/" in response.url
-
-
-def test_selftest_requires_staff_permission(client):
-    """Test that only staff users can access the selftest page."""
-    selftest_url = reverse("admin:selftest")
-
-    User.objects.create_user(
-        username="user",
-        email="user@example.com",
-        password="user123",
-    )
-    client.login(username="user", password="user123")
-
-    response = client.get(selftest_url)
-
-    # Regular users should be redirected
-    assert response.status_code == 302
-
-
-def test_selftest_accessible_by_admin(client):
-    """Test that admin users can access the selftest page."""
-    selftest_url = reverse("admin:selftest")
-
-    User.objects.create_superuser(
-        username="admin",
-        email="admin@example.com",
-        password="admin123",
-    )
-    client.login(username="admin", password="admin123")
-
-    response = client.get(selftest_url)
-
-    assert response.status_code == 200
-    assert b"System Self-Tests" in response.content
-
-
-def test_selftest_displays_available_tests(client):
-    """Test that available tests are displayed when not running."""
-    selftest_url = reverse("admin:selftest")
-
-    User.objects.create_superuser(
-        username="admin",
-        email="admin@example.com",
-        password="admin123",
-    )
-    client.login(username="admin", password="admin123")
-
-    response = client.get(selftest_url)
-
-    assert response.status_code == 200
-    assert b"Available Tests" in response.content
-    assert b"Run All Tests" in response.content
-
-
-@patch("core.selftests.registry.run_all")
-def test_selftest_runs_tests(mock_run_all, client):
-    """Test that tests are executed when run=true."""
-    selftest_url = reverse("admin:selftest")
-
-    # Mock the test results
-    mock_run_all.return_value = [
-        SelfTestResult(
-            name="Test 1",
-            success=True,
-            message="Success",
-            duration_ms=10.0,
-        ),
-        SelfTestResult(
-            name="Test 2",
-            success=False,
-            message="Failed",
-            duration_ms=20.0,
-        ),
-    ]
-
-    User.objects.create_superuser(
-        username="admin",
-        email="admin@example.com",
-        password="admin123",
-    )
-    client.login(username="admin", password="admin123")
-
-    response = client.get(selftest_url, {"run": "true"})
-
-    assert response.status_code == 200
-    assert b"Test Results" in response.content
-    assert b"Test 1" in response.content
-    assert b"Test 2" in response.content
-    mock_run_all.assert_called_once()
-
-
-@patch("core.selftests.registry.run_all")
-def test_selftest_displays_success_status(mock_run_all, client):
-    """Test that success status is displayed correctly."""
-    selftest_url = reverse("admin:selftest")
-
-    mock_run_all.return_value = [
-        SelfTestResult(
-            name="Test 1",
-            success=True,
-            message="Success",
-            duration_ms=10.0,
-        ),
-    ]
-
-    User.objects.create_superuser(
-        username="admin",
-        email="admin@example.com",
-        password="admin123",
-    )
-    client.login(username="admin", password="admin123")
-
-    response = client.get(selftest_url, {"run": "true"})
-
-    assert response.status_code == 200
-    assert b"All tests passed successfully" in response.content
-
-
-@patch("core.selftests.registry.run_all")
-def test_selftest_displays_failure_status(mock_run_all, client):
-    """Test that failure status is displayed correctly."""
-    selftest_url = reverse("admin:selftest")
-
-    mock_run_all.return_value = [
-        SelfTestResult(
-            name="Test 1",
-            success=False,
-            message="Failed",
-            duration_ms=10.0,
-        ),
-    ]
-
-    User.objects.create_superuser(
-        username="admin",
-        email="admin@example.com",
-        password="admin123",
-    )
-    client.login(username="admin", password="admin123")
-
-    response = client.get(selftest_url, {"run": "true"})
-
-    assert response.status_code == 200
-    assert b"Some tests failed" in response.content

src/backend/core/tests/test_api_documents_delete.py

@@ -1,384 +0,0 @@
-"""Tests for deleting documents from OpenSearch over the API"""
-
-import opensearchpy
-import pytest
-import responses
-from rest_framework.test import APIClient
-
-from core import factories
-from core.services.opensearch import opensearch_client
-from core.utils import prepare_index
-
-from .utils import build_authorization_bearer, setup_oicd_resource_server
-
-pytestmark = pytest.mark.django_db
-
-
-def test_api_documents_delete_anonymous():
-    """Anonymous requests should not be allowed to delete documents."""
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": "service-name", "document_ids": ["doc1"]},
-        format="json",
-    )
-
-    assert response.status_code == 401
-
-
-@responses.activate
-def test_api_documents_delete_wrong_service_name(settings):
-    """Requests with a wrong service name should return 400 Bad Request."""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": "wrong-service", "document_ids": ["0"]},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 400
-    assert response.json()["detail"] == "Invalid request."
-
-
-@responses.activate
-def test_api_documents_delete_success(settings):
-    """Authenticated users should be able to delete documents they have access to."""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-
-    service = factories.ServiceFactory()
-    # Create documents user has access to
-    documents = factories.DocumentSchemaFactory.build_batch(3, users=["user_sub"])
-    prepare_index(service.index_name, documents)
-    document_to_delete_ids = [doc["id"] for doc in documents[:2]]
-
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": service.name, "document_ids": document_to_delete_ids},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 200
-    assert response.json()["nb-deleted-documents"] == 2
-    assert response.json()["undeleted-document-ids"] == []
-
-    opensearch_client_ = opensearch_client()
-    for document in documents:
-        if document["id"] in document_to_delete_ids:
-            with pytest.raises(opensearchpy.exceptions.NotFoundError):
-                opensearch_client_.get(index=service.index_name, id=document["id"])
-        else:
-            doc = opensearch_client_.get(index=service.index_name, id=document["id"])
-            assert doc["found"]
-
-
-@responses.activate
-def test_api_documents_delete_no_access(settings):
-    """Users should not be able to delete documents they don't have access to."""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    service = factories.ServiceFactory()
-    # Create documents where user_sub does NOT have access
-    documents = factories.DocumentSchemaFactory.build_batch(2, users=["other_sub"])
-    prepare_index(service.index_name, documents)
-
-    document_ids = [doc["id"] for doc in documents]
-
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": service.name, "document_ids": document_ids},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 200
-    assert response.json()["nb-deleted-documents"] == 0
-    assert set(response.json()["undeleted-document-ids"]) == set(document_ids)
-
-    # Verify documents not deleted
-    opensearch_client_ = opensearch_client()
-    for doc_id in document_ids:
-        doc = opensearch_client_.get(index=service.index_name, id=doc_id)
-        assert doc["found"]
-
-
-@responses.activate
-def test_api_documents_delete_mixed_access(settings):
-    """Deleting a mix of owned and non-owned documents should only delete owned ones."""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-
-    service = factories.ServiceFactory()
-    # Create documents with different access
-    owned_documents = factories.DocumentSchemaFactory.build_batch(2, users=["user_sub"])
-    other_documents = factories.DocumentSchemaFactory.build_batch(
-        2, users=["other_user"]
-    )
-    prepare_index(service.index_name, owned_documents + other_documents)
-
-    owned_document_ids = [doc["id"] for doc in owned_documents]
-    other_document_ids = [doc["id"] for doc in other_documents]
-    non_existing_document_ids = ["non-existent-1", "non-existent-2"]
-
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {
-            "service": service.name,
-            "document_ids": owned_document_ids
-            + other_document_ids
-            + non_existing_document_ids,
-        },
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 200
-    assert response.json()["nb-deleted-documents"] == 2
-    assert set(response.json()["undeleted-document-ids"]) == set(
-        other_document_ids + non_existing_document_ids
-    )
-
-    # Verify only owned documents are deleted
-    opensearch_client_ = opensearch_client()
-    for document_id in owned_document_ids:
-        with pytest.raises(opensearchpy.exceptions.NotFoundError):
-            opensearch_client_.get(index=service.index_name, id=document_id)
-
-    for document_id in other_document_ids:
-        document = opensearch_client_.get(index=service.index_name, id=document_id)
-        assert document["found"]
-
-
-@responses.activate
-def test_api_documents_delete_invalid_params(settings):
-    """Requests with invalid parameters should return 400 Bad Request."""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    service = factories.ServiceFactory()
-
-    # Missing both document_ids and tags
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {
-            "service": service.name,
-        },
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 400
-    assert (
-        response.json()[0]["msg"]
-        == "Value error, At least one of 'document_ids' or 'tags' must be provided"
-    )
-
-    # Empty document_ids and no tags
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": service.name, "document_ids": []},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 400
-    assert (
-        response.json()[0]["msg"]
-        == "Value error, At least one of 'document_ids' or 'tags' must be provided"
-    )
-
-    # Both empty
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": service.name, "document_ids": [], "tags": []},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 400
-    assert (
-        response.json()[0]["msg"]
-        == "Value error, At least one of 'document_ids' or 'tags' must be provided"
-    )
-
-    # Missing service
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"document_ids": ["doc1"]},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 400
-
-
-@responses.activate
-def test_api_documents_delete_nonexistent_documents(settings):
-    """
-    Deleting non-existent documents should not raise an error
-    and return the list of undeleted ids.
-    """
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    service = factories.ServiceFactory()
-    # Create index but with no documents
-    prepare_index(service.index_name, [])
-
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": service.name, "document_ids": ["non-existent-id"]},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 200
-    assert response.json()["nb-deleted-documents"] == 0
-    assert response.json()["undeleted-document-ids"] == ["non-existent-id"]
-
-
-@responses.activate
-def test_api_documents_delete_by_single_tag(settings):
-    """Users should be able to delete documents by tags."""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    service = factories.ServiceFactory()
-
-    document_to_deletes = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["delete-tag", "keep-tag-1"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["delete-tag", "keep-tag-2"]
-        ),
-    ]
-    document_to_keep = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["keep-tag-1", "keep-tag-2"]
-        ),
-        factories.DocumentSchemaFactory.build(users=["user_sub"], tags=["keep-tag-1"]),
-        factories.DocumentSchemaFactory.build(
-            users=["other_user_sub"], tags=["delete-tag"]
-        ),
-    ]
-    prepare_index(service.index_name, document_to_deletes + document_to_keep)
-
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": service.name, "tags": ["delete-tag"]},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 200
-    assert response.json()["nb-deleted-documents"] == 2
-    assert response.json()["undeleted-document-ids"] == []
-
-    opensearch_client_ = opensearch_client()
-    for document in document_to_deletes:
-        with pytest.raises(opensearchpy.exceptions.NotFoundError):
-            opensearch_client_.get(index=service.index_name, id=document["id"])
-
-    for document in document_to_keep:
-        doc = opensearch_client_.get(index=service.index_name, id=document["id"])
-        assert doc["found"]
-
-
-@responses.activate
-def test_api_documents_delete_by_multiple_tags(settings):
-    """Users should be able to delete documents matching any of multiple tags."""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    service = factories.ServiceFactory()
-
-    document_to_deletes = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["delete-tag-1", "keep-tag-1"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["delete-tag-1", "delete-tag-2"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["delete-tag-2"]
-        ),
-    ]
-    document_to_keep = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["keep-tag-1", "keep-tag-2"]
-        ),
-        factories.DocumentSchemaFactory.build(users=["user_sub"], tags=["keep-tag-1"]),
-        factories.DocumentSchemaFactory.build(
-            users=["other_user_sub"], tags=["delete-tag-1"]
-        ),
-    ]
-    prepare_index(service.index_name, document_to_deletes + document_to_keep)
-
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {"service": service.name, "tags": ["delete-tag-1", "delete-tag-2"]},
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 200
-    assert response.json()["nb-deleted-documents"] == 3
-    assert response.json()["undeleted-document-ids"] == []
-
-    opensearch_client_ = opensearch_client()
-    for document in document_to_deletes:
-        with pytest.raises(opensearchpy.exceptions.NotFoundError):
-            opensearch_client_.get(index=service.index_name, id=document["id"])
-
-    for document in document_to_keep:
-        doc = opensearch_client_.get(index=service.index_name, id=document["id"])
-        assert doc["found"]
-
-
-@responses.activate
-def test_api_documents_delete_by_ids_and_tags(settings):
-    """Users should be able to delete documents by both IDs and tags (AND logic)."""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    service = factories.ServiceFactory()
-
-    document_delete_by_tag_and_id = factories.DocumentSchemaFactory.build(
-        users=["user_sub"], tags=["delete-tag"]
-    )
-    document_delete_by_tag_keep_by_id = factories.DocumentSchemaFactory.build(
-        users=["user_sub"], tags=["delete-tag"]
-    )
-    document_keep_by_tag_delete_by_id = factories.DocumentSchemaFactory.build(
-        users=["user_sub"]
-    )
-
-    prepare_index(
-        service.index_name,
-        [
-            document_delete_by_tag_and_id,
-            document_delete_by_tag_keep_by_id,
-            document_keep_by_tag_delete_by_id,
-        ],
-    )
-
-    response = APIClient().post(
-        "/api/v1.0/documents/delete/",
-        {
-            "service": service.name,
-            "document_ids": [
-                document_delete_by_tag_and_id["id"],
-                document_keep_by_tag_delete_by_id["id"],
-            ],
-            "tags": ["delete-tag"],
-        },
-        format="json",
-        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-    )
-
-    assert response.status_code == 200
-    assert response.json()["nb-deleted-documents"] == 1
-    assert response.json()["undeleted-document-ids"] == [
-        document_keep_by_tag_delete_by_id["id"]
-    ]
-
-    opensearch_client_ = opensearch_client()
-    with pytest.raises(opensearchpy.exceptions.NotFoundError):
-        opensearch_client_.get(
-            index=service.index_name, id=document_delete_by_tag_and_id["id"]
-        )
-
-    doc = opensearch_client_.get(
-        index=service.index_name, id=document_delete_by_tag_keep_by_id["id"]
-    )
-    assert doc["found"]

src/backend/core/tests/test_api_documents_index_bulk.py

@@ -6,11 +6,9 @@ from unittest import mock
 from django.utils import timezone
 
 import pytest
-from opensearchpy import NotFoundError
 from rest_framework.test import APIClient
 
-from core import factories
-from core.services import opensearch
+from core import factories, opensearch
 
 pytestmark = pytest.mark.django_db
 
@@ -44,7 +42,7 @@ def test_api_documents_index_bulk_invalid_token():
 
 def test_api_documents_index_bulk_success():
     """A registered service should be able to index documents in bulk with a valid token."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
     response = APIClient().post(
@@ -54,19 +52,21 @@ def test_api_documents_index_bulk_success():
         format="json",
     )
 
-    assert response.status_code == 201
+    assert response.status_code == 207
     responses = response.json()
     assert [d["status"] for d in responses] == ["success"] * 3
 
 
 def test_api_documents_index_bulk_ensure_index():
-    """A registered service should be created the opensearch index if needed."""
-    opensearch_client_ = opensearch.opensearch_client()
-    service = factories.ServiceFactory()
+    """A registered service should be create the opensearch index if need."""
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
-    with pytest.raises(NotFoundError):
-        opensearch_client_.indices.get(index=service.index_name)
+    # Delete the index
+    opensearch.client.indices.delete(index="*test*")
+
+    with pytest.raises(opensearch.NotFoundError):
+        opensearch.client.indices.get(index="test-service")
 
     response = APIClient().post(
         "/api/v1.0/documents/index/",
@@ -75,13 +75,13 @@ def test_api_documents_index_bulk_ensure_index():
         format="json",
     )
 
-    assert response.status_code == 201
+    assert response.status_code == 207
     responses = response.json()
     assert len(responses) == 3
     assert [d["status"] for d in responses] == ["success"] * 3
 
     # The index has been rebuilt
-    opensearch_client_.indices.get(index=service.index_name)
+    opensearch.client.indices.get(index="test-service")
 
 
 @pytest.mark.parametrize(
@@ -200,7 +200,7 @@ def test_api_documents_index_bulk_invalid_document(
     field, invalid_value, error_type, error_message
 ):
     """Test bulk document indexing with various invalid fields."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
     # Modify the first document with the invalid value for the specified field
@@ -240,7 +240,7 @@ def test_api_documents_index_bulk_invalid_document(
 )
 def test_api_documents_index_bulk_required(field):
     """Test bulk document indexing with a required field missing."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
     del documents[0][field]
@@ -272,7 +272,7 @@ def test_api_documents_index_bulk_required(field):
 )
 def test_api_documents_index_bulk_default(field, default_value):
     """Test bulk document indexing while removing optional fields that have default values."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
     del documents[0][field]
@@ -284,19 +284,19 @@ def test_api_documents_index_bulk_default(field, default_value):
         format="json",
     )
 
-    assert response.status_code == 201
+    assert response.status_code == 207
     responses = response.json()
     assert [d["status"] for d in responses] == ["success"] * 3
 
-    indexed_document = opensearch.opensearch_client().get(
-        index=service.index_name, id=responses[0]["_id"]
+    indexed_document = opensearch.client.get(
+        index=service.name, id=responses[0]["_id"]
     )["_source"]
     assert indexed_document[field] == default_value
 
 
 def test_api_documents_index_bulk_updated_at_before_created_at():
     """Test bulk document indexing with updated_at before created_at."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
     documents[0]["updated_at"] = documents[0]["created_at"] - datetime.timedelta(
@@ -330,7 +330,7 @@ def test_api_documents_index_bulk_updated_at_before_created_at():
 )
 def test_api_documents_index_bulk_datetime_future(field):
     """Test bulk document indexing with datetimes in the future."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
     now = timezone.now()
@@ -358,7 +358,7 @@ def test_api_documents_index_bulk_datetime_future(field):
 
 def test_api_documents_index_empty_content_check():
     """Test bulk document indexing with both empty title & content."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
     documents[0]["content"] = ""
@@ -386,10 +386,10 @@ def test_api_documents_index_empty_content_check():
 
 def test_api_documents_index_opensearch_errors():
     """Test bulk document indexing errors"""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = factories.DocumentSchemaFactory.build_batch(3)
 
-    with mock.patch.object(opensearch.opensearch_client(), "bulk") as mock_bulk:
+    with mock.patch.object(opensearch.client, "bulk") as mock_bulk:
         mock_bulk.return_value = {
             "items": [
                 {"index": {"status": 201}},
@@ -409,7 +409,7 @@ def test_api_documents_index_opensearch_errors():
             format="json",
         )
 
-    assert response.status_code == 201
+    assert response.status_code == 207
     responses = response.json()
     assert responses == [
         {

src/backend/core/tests/test_api_documents_index_single.py

@@ -5,24 +5,13 @@ import datetime
 from django.utils import timezone
 
 import pytest
-import responses
-from opensearchpy import NotFoundError
 from rest_framework.test import APIClient
 
-from core import factories
-from core.services import opensearch
-from core.tests.mock import albert_embedding_response
-from core.tests.utils import enable_hybrid_search
+from core import factories, opensearch
 
 pytestmark = pytest.mark.django_db
 
 
-@pytest.fixture(autouse=True)
-def clear_caches():
-    """Clear caches and delete search pipeline before each test"""
-    opensearch.check_hybrid_search_enabled.cache_clear()
-
-
 def test_api_documents_index_single_anonymous():
     """Anonymous requests should not be allowed to index documents."""
     document = factories.DocumentSchemaFactory.build()
@@ -50,77 +39,9 @@ def test_api_documents_index_single_invalid_token():
     assert response.json() == {"detail": "Invalid token."}
 
 
-@responses.activate
-def test_api_documents_index_single_hybrid_enabled_success(settings):
-    """
-    A registered service should be able to index document with a valid token.
-    If hybrid search is enabled, the documents are chunked and embedded.
-    """
-    service = factories.ServiceFactory()
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    document = factories.DocumentSchemaFactory.build()
-    document["content"] = (
-        "a long text to embed." * 100
-    )  # Ensure content is long enough for chunking
-
-    response = APIClient().post(
-        "/api/v1.0/documents/index/",
-        document,
-        HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
-        format="json",
-    )
-
-    assert response.status_code == 201
-    assert response.json()["_id"] == str(document["id"])
-
-    new_indexed_document = opensearch.opensearch_client().get(
-        index=service.index_name, id=str(document["id"])
-    )
-    assert new_indexed_document["_version"] == 1
-
-    assert (
-        new_indexed_document["_source"]["title.en"] == document["title"].strip().lower()
-    )
-    assert new_indexed_document["_source"]["content.en"] == document["content"]
-    # only the english fields are indexed
-    assert not "content.fr" in new_indexed_document["_source"]
-
-    # check embedding
-    assert (
-        new_indexed_document["_source"]["chunks"][0]["embedding"]
-        == albert_embedding_response.response["data"][0]["embedding"]
-    )
-    assert (
-        new_indexed_document["_source"]["embedding_model"]
-        == settings.EMBEDDING_API_MODEL_NAME
-    )
-    # Check that the document has been chunked correctly
-    assert (
-        len(new_indexed_document["_source"]["chunks"])
-        == int(
-            len(document["content"]) / (settings.CHUNK_SIZE - settings.CHUNK_OVERLAP)
-        )
-        + 1
-    )
-    for chunk in new_indexed_document["_source"]["chunks"]:
-        assert (
-            chunk["embedding"]
-            == albert_embedding_response.response["data"][0]["embedding"]
-        )
-        assert chunk["content"] in document["content"]
-        assert len(chunk["content"]) < len(document["content"])
-
-
-def test_api_documents_index_language_params():
-    """language_code query param should control which language is indexed."""
-    service = factories.ServiceFactory()
+def test_api_documents_index_single_success():
+    """A registered service should be able to index document with a valid token."""
+    service = factories.ServiceFactory(name="test-service")
     document = factories.DocumentSchemaFactory.build()
 
     response = APIClient().post(
@@ -133,109 +54,17 @@ def test_api_documents_index_language_params():
     assert response.status_code == 201
     assert response.json()["_id"] == str(document["id"])
 
-    new_indexed_document = opensearch.opensearch_client().get(
-        index=service.index_name, id=str(document["id"])
-    )
-    language_code = "en"
-    assert (
-        new_indexed_document["_source"][f"title.{language_code}"]
-        == document["title"].strip().lower()
-    )
-    assert (
-        new_indexed_document["_source"][f"content.{language_code}"]
-        == document["content"]
-    )
-    other_language_code = "fr"
-    # only the requested language is indexed
-    assert not f"content.{other_language_code}" in new_indexed_document["_source"]
 
-
-def test_api_documents_index_and_reindex_same_document():
-    """
-    Indexing the same document twice should update it.
-    If the detected language changes the new language code should be used and the
-    former language code should not be present anymore.
-    """
-    service = factories.ServiceFactory()
+def test_api_documents_index_bulk_ensure_index():
+    """A registered service should be create the opensearch index if need."""
+    service = factories.ServiceFactory(name="test-service")
     document = factories.DocumentSchemaFactory.build()
 
-    # First indexing with unrecognized language title
-    document["title"] = "planning"
-    APIClient().post(
-        "/api/v1.0/documents/index/",
-        document,
-        HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
-        format="json",
-    )
+    # Delete the index
+    opensearch.client.indices.delete(index="*test*")
 
-    new_indexed_document = opensearch.opensearch_client().get(
-        index=service.index_name, id=str(document["id"])
-    )
-    assert new_indexed_document["_version"] == 1
-    assert (
-        new_indexed_document["_source"]["title.und"]
-        == document["title"].strip().lower()
-    )
-    assert new_indexed_document["_source"]["content.und"] == document["content"].strip()
-
-    # Index the same document with a french content
-    document["content"] = "du contenu en francais"
-    APIClient().post(
-        "/api/v1.0/documents/index/",
-        document,
-        HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
-        format="json",
-    )
-
-    new_indexed_document = opensearch.opensearch_client().get(
-        index=service.index_name, id=str(document["id"])
-    )
-    assert new_indexed_document["_version"] == 2
-    # the document is detected as french
-    assert (
-        new_indexed_document["_source"]["title.fr"] == document["title"].strip().lower()
-    )
-    assert new_indexed_document["_source"]["content.fr"] == document["content"]
-    # und field are removed
-    assert "title.und" not in new_indexed_document["_source"]
-    assert "content.und" not in new_indexed_document["_source"]
-
-
-def test_api_documents_index_single_hybrid_disabled_success():
-    """If hybrid search is not enabled, the indexing should have an embedding equal to None."""
-    service = factories.ServiceFactory()
-    document = factories.DocumentSchemaFactory.build()
-    opensearch.check_hybrid_search_enabled.cache_clear()
-
-    response = APIClient().post(
-        "/api/v1.0/documents/index/",
-        document,
-        HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
-        format="json",
-    )
-
-    assert response.status_code == 201
-    assert response.json()["_id"] == str(document["id"])
-
-    new_indexed_document = opensearch.opensearch_client().get(
-        index=service.index_name, id=str(document["id"])
-    )
-    assert new_indexed_document["_version"] == 1
-    assert (
-        new_indexed_document["_source"]["title.en"] == document["title"].strip().lower()
-    )
-    assert new_indexed_document["_source"]["content.en"] == document["content"]
-    assert new_indexed_document["_source"]["chunks"] is None
-
-
-def test_api_documents_index_single_ensure_index(settings):
-    """A registered service should be created the opensearch index if needed."""
-    service = factories.ServiceFactory()
-    document = factories.DocumentSchemaFactory.build()
-    opensearch_client_ = opensearch.opensearch_client()
-
-    with pytest.raises(NotFoundError):
-        opensearch_client_.indices.get(index=service.index_name)
+    with pytest.raises(opensearch.NotFoundError):
+        opensearch.client.indices.get(index="test-service")
 
     response = APIClient().post(
         "/api/v1.0/documents/index/",
@@ -248,159 +77,32 @@ def test_api_documents_index_single_ensure_index(settings):
     assert response.json()["_id"] == str(document["id"])
 
     # The index has been rebuilt
-    data = opensearch_client_.indices.get(index=service.index_name)
+    data = opensearch.client.indices.get(index="test-service")
 
-    assert data[service.index_name]["mappings"] == {
+    assert data["test-service"]["mappings"] == {
         "dynamic": "strict",
         "properties": {
-            "chunks": {
-                "type": "nested",
-                "properties": {
-                    "content": {"type": "text"},
-                    "embedding": {
-                        "type": "knn_vector",
-                        "dimension": settings.EMBEDDING_DIMENSION,
-                        "method": {
-                            "engine": "lucene",
-                            "space_type": "l2",
-                            "name": "hnsw",
-                            "parameters": {},
-                        },
-                    },
-                    "index": {"type": "integer"},
+            "id": {"type": "keyword"},
+            "title": {
+                "type": "keyword",  # Primary field for exact matches and sorting
+                "fields": {
+                    "text": {"type": "text"}  # Sub-field for full-text search
                 },
             },
-            "content": {
-                "properties": {
-                    "de": {
-                        "type": "text",
-                        "fields": {
-                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
-                        },
-                        "analyzer": "german_analyzer",
-                    },
-                    "en": {
-                        "type": "text",
-                        "fields": {
-                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
-                        },
-                        "analyzer": "english_analyzer",
-                    },
-                    "fr": {
-                        "type": "text",
-                        "fields": {
-                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
-                        },
-                        "analyzer": "french_analyzer",
-                    },
-                    "nl": {
-                        "type": "text",
-                        "fields": {
-                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
-                        },
-                        "analyzer": "dutch_analyzer",
-                    },
-                    "und": {
-                        "type": "text",
-                        "fields": {
-                            "trigrams": {"type": "text", "analyzer": "trigram_analyzer"}
-                        },
-                        "analyzer": "undetermined_language_analyzer",
-                    },
-                }
-            },
-            "created_at": {"type": "date"},
             "depth": {"type": "integer"},
-            "embedding_model": {"type": "keyword"},
-            "groups": {"type": "keyword"},
-            "id": {"type": "keyword"},
-            "is_active": {"type": "boolean"},
-            "numchild": {"type": "integer"},
-            "path": {"type": "keyword", "fields": {"text": {"type": "text"}}},
-            "reach": {"type": "keyword"},
-            "size": {"type": "long"},
-            "tags": {"type": "keyword"},
-            "title": {
-                "properties": {
-                    "de": {
-                        "type": "keyword",
-                        "fields": {
-                            "text": {
-                                "type": "text",
-                                "fields": {
-                                    "trigrams": {
-                                        "type": "text",
-                                        "analyzer": "trigram_analyzer",
-                                    }
-                                },
-                                "analyzer": "german_analyzer",
-                            }
-                        },
-                    },
-                    "en": {
-                        "type": "keyword",
-                        "fields": {
-                            "text": {
-                                "type": "text",
-                                "fields": {
-                                    "trigrams": {
-                                        "type": "text",
-                                        "analyzer": "trigram_analyzer",
-                                    }
-                                },
-                                "analyzer": "english_analyzer",
-                            }
-                        },
-                    },
-                    "fr": {
-                        "type": "keyword",
-                        "fields": {
-                            "text": {
-                                "type": "text",
-                                "fields": {
-                                    "trigrams": {
-                                        "type": "text",
-                                        "analyzer": "trigram_analyzer",
-                                    }
-                                },
-                                "analyzer": "french_analyzer",
-                            }
-                        },
-                    },
-                    "nl": {
-                        "type": "keyword",
-                        "fields": {
-                            "text": {
-                                "type": "text",
-                                "fields": {
-                                    "trigrams": {
-                                        "type": "text",
-                                        "analyzer": "trigram_analyzer",
-                                    }
-                                },
-                                "analyzer": "dutch_analyzer",
-                            }
-                        },
-                    },
-                    "und": {
-                        "type": "keyword",
-                        "fields": {
-                            "text": {
-                                "type": "text",
-                                "fields": {
-                                    "trigrams": {
-                                        "type": "text",
-                                        "analyzer": "trigram_analyzer",
-                                    }
-                                },
-                                "analyzer": "undetermined_language_analyzer",
-                            }
-                        },
-                    },
-                }
+            "path": {
+                "type": "keyword",
+                "fields": {"text": {"type": "text"}},
             },
+            "numchild": {"type": "integer"},
+            "content": {"type": "text"},
+            "created_at": {"type": "date"},
             "updated_at": {"type": "date"},
+            "size": {"type": "long"},
             "users": {"type": "keyword"},
+            "groups": {"type": "keyword"},
+            "reach": {"type": "keyword"},
+            "is_active": {"type": "boolean"},
         },
     }
 
@@ -521,7 +223,7 @@ def test_api_documents_index_single_invalid_document(
     field, invalid_value, error_type, error_message
 ):
     """Test document indexing with various invalid fields."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     document = factories.DocumentSchemaFactory.build()
 
     # Modify the document with the invalid value for the specified field
@@ -556,7 +258,7 @@ def test_api_documents_index_single_invalid_document(
 )
 def test_api_documents_index_single_required(field):
     """Test document indexing with a required field missing."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     document = factories.DocumentSchemaFactory.build()
 
     del document[field]
@@ -583,7 +285,7 @@ def test_api_documents_index_single_required(field):
 )
 def test_api_documents_index_single_default(field, default_value):
     """Test document indexing while removing optional fields that have default values."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     document = factories.DocumentSchemaFactory.build()
 
     del document[field]
@@ -598,15 +300,15 @@ def test_api_documents_index_single_default(field, default_value):
     assert response.status_code == 201
     assert response.json()["_id"] == str(document["id"])
 
-    indexed_document = opensearch.opensearch_client().get(
-        index=service.index_name, id=str(document["id"])
+    indexed_document = opensearch.client.get(
+        index=service.name, id=str(document["id"])
     )["_source"]
     assert indexed_document[field] == default_value
 
 
 def test_api_documents_index_single_udpated_at_before_created():
     """Test document indexing with updated_at before created_at."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     document = factories.DocumentSchemaFactory.build()
 
     document["updated_at"] = document["created_at"] - datetime.timedelta(seconds=1)
@@ -632,7 +334,7 @@ def test_api_documents_index_single_udpated_at_before_created():
 )
 def test_api_documents_index_single_datetime_future(field):
     """Test document indexing with datetimes in the future."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     document = factories.DocumentSchemaFactory.build()
 
     now = timezone.now()
@@ -652,7 +354,7 @@ def test_api_documents_index_single_datetime_future(field):
 
 def test_api_documents_index_empty_content_check():
     """Test document indexing with both empty title & content."""
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     document = factories.DocumentSchemaFactory.build()
 
     document["content"] = ""

src/backend/core/tests/test_api_documents_search_access_control.py

@@ -10,32 +10,24 @@ import responses
 from rest_framework.test import APIClient
 
 from core import enums, factories
-from core.services.opensearch import opensearch_client
-from core.utils import prepare_index
 
-from .mock import albert_embedding_response
 from .utils import (
     build_authorization_bearer,
+    delete_test_indices,
+    prepare_index,
     setup_oicd_resource_server,
 )
 
 pytestmark = pytest.mark.django_db
 
 
-@responses.activate
-def test_api_documents_search_access_control_anonymous(settings):
+def test_api_documents_search_access_control_anonymous():
     """Anonymous users should not be allowed to search documents even public."""
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents = []
     for reach in enums.ReachEnum:
         documents.extend(factories.DocumentSchemaFactory.build_batch(3, reach=reach))
-    prepare_index(service.index_name, documents)
+    prepare_index(service.name, documents)
 
     response = APIClient().post("/api/v1.0/documents/search/?q=*")
 
@@ -51,16 +43,10 @@ def test_api_documents_search_access_control(settings):
     - only configured services providers are allowed (e.g docs)
     (groups is not yet implemnted)
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub")
     token = build_authorization_bearer()
 
-    service = factories.ServiceFactory()
+    service = factories.ServiceFactory(name="test-service")
     documents_reach = factories.DocumentSchemaFactory.build_batch(6)
     documents_open = [
         doc for doc in documents_reach if doc["reach"] in ["authenticated", "public"]
@@ -70,7 +56,7 @@ def test_api_documents_search_access_control(settings):
     )
     expected_ids = [doc["id"] for doc in documents_open + documents_user]
 
-    prepare_index(service.index_name, documents_user + documents_reach)
+    prepare_index(service.name, documents_user + documents_reach)
 
     response = APIClient().post(
         "/api/v1.0/documents/search/",
@@ -104,16 +90,10 @@ def test_api_documents_search_access__only_visited_public(
     Authenticated users should only see documents with reach="public"
     that are in "visited" list.
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs")
     token = build_authorization_bearer()
 
-    service = factories.ServiceFactory(client_id="docs")
+    service = factories.ServiceFactory(name="test-service", client_id="docs")
 
     docs = [
         factories.DocumentSchemaFactory(
@@ -122,7 +102,7 @@ def test_api_documents_search_access__only_visited_public(
         for doc_id in doc_ids
     ]
 
-    prepare_index(service.index_name, docs)
+    prepare_index(service.name, docs)
 
     response = APIClient().post(
         "/api/v1.0/documents/search/",
@@ -141,16 +121,10 @@ def test_api_documents_search_access__any_owner_public(settings):
     Authenticated users should only see documents with reach="public"
     that are in "visited" list.
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs")
     token = build_authorization_bearer()
 
-    service = factories.ServiceFactory(client_id="docs")
+    service = factories.ServiceFactory(name="test-service", client_id="docs")
 
     docs = factories.DocumentSchemaFactory.build_batch(
         6,
@@ -164,7 +138,7 @@ def test_api_documents_search_access__any_owner_public(settings):
         users=["other_sub"],
     )
 
-    prepare_index(service.index_name, docs + other_docs)
+    prepare_index(service.name, docs + other_docs)
 
     expected = [d["id"] for d in docs]
 
@@ -185,17 +159,11 @@ def test_api_documents_search_access__services(settings):
     Authenticated users should only see documents of audience
     service providers (e.g docs)
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub", audience="a-client")
     token = build_authorization_bearer()
 
-    service_a = factories.ServiceFactory(client_id="a-client")
-    service_b = factories.ServiceFactory(client_id="b-client")
+    service_a = factories.ServiceFactory(name="test-index-a", client_id="a-client")
+    service_b = factories.ServiceFactory(name="test-index-b", client_id="b-client")
 
     service_a_docs = factories.DocumentSchemaFactory.build_batch(
         3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"]
@@ -206,8 +174,8 @@ def test_api_documents_search_access__services(settings):
 
     expected_ids = [doc["id"] for doc in service_a_docs]
 
-    prepare_index(service_a.index_name, service_a_docs)
-    prepare_index(service_b.index_name, service_b_docs)
+    prepare_index(service_a.name, service_a_docs)
+    prepare_index(service_b.name, service_b_docs, cleanup=False)
 
     response = APIClient().post(
         "/api/v1.0/documents/search/",
@@ -225,17 +193,11 @@ def test_api_documents_search_access__missing_index(settings):
     """
     When the service has no opensearch index, returns an empty list.
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub", audience="a-client")
     token = build_authorization_bearer()
     factories.ServiceFactory(name="test-index-a", client_id="a-client")
 
-    opensearch_client.cache_clear()
+    delete_test_indices()
 
     # a-client has no index. ignore it.
     response = APIClient().post(
@@ -255,12 +217,6 @@ def test_api_documents_search_access__related_services(settings):
     Authenticated users should only see documents of audience
     service providers and its related services (e.g drive + docs)
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub", audience="c-client")
     token = build_authorization_bearer()
 
@@ -282,9 +238,9 @@ def test_api_documents_search_access__related_services(settings):
 
     expected_ids = [doc["id"] for doc in service_a_docs + service_c_docs]
 
-    prepare_index(service_a.index_name, service_a_docs)
-    prepare_index(service_b.index_name, service_b_docs)
-    prepare_index(service_c.index_name, service_c_docs)
+    prepare_index(service_a.name, service_a_docs)
+    prepare_index(service_b.name, service_b_docs, cleanup=False)
+    prepare_index(service_c.name, service_c_docs, cleanup=False)
 
     response = APIClient().post(
         "/api/v1.0/documents/search/",
@@ -302,12 +258,6 @@ def test_api_documents_search_access__related_missing_index(settings):
     """
     When the service has no opensearch index, returns the related services data.
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub", audience="a-client")
     token = build_authorization_bearer()
 
@@ -326,8 +276,8 @@ def test_api_documents_search_access__related_missing_index(settings):
 
     expected_ids = [doc["id"] for doc in service_c_docs]
 
-    prepare_index(service_b.index_name, service_b_docs)
-    prepare_index(service_c.index_name, service_c_docs)
+    prepare_index(service_b.name, service_b_docs)
+    prepare_index(service_c.name, service_c_docs, cleanup=False)
 
     # a-client has no index. ignore it.
     response = APIClient().post(
@@ -348,12 +298,6 @@ def test_api_documents_search_access__request_services(settings):
     from requested services : 'services' parameter.
     Raise 400 error if not all requested services are authorized.
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub", audience="c-client")
     token = build_authorization_bearer()
 
@@ -373,9 +317,9 @@ def test_api_documents_search_access__request_services(settings):
 
     expected_ids = [doc["id"] for doc in service_c_docs]
 
-    prepare_index(service_a.index_name, service_a_docs)
-    prepare_index(service_b.index_name, service_b_docs)
-    prepare_index(service_c.index_name, service_c_docs)
+    prepare_index(service_a.name, service_a_docs)
+    prepare_index(service_b.name, service_b_docs, cleanup=False)
+    prepare_index(service_c.name, service_c_docs, cleanup=False)
 
     response = APIClient().post(
         "/api/v1.0/documents/search/",
@@ -395,7 +339,7 @@ def test_api_documents_search_access__request_services(settings):
     )
 
     assert response.status_code == 400
-    assert response.json() == {"detail": "Invalid request."}
+    assert response.json() == {"detail": "Some requested services are not available"}
 
 
 @responses.activate
@@ -419,7 +363,7 @@ def test_api_documents_search_access__request_inactive_services(settings):
     )
 
     assert response.status_code == 400
-    assert response.json() == {"detail": "Invalid request."}
+    assert response.json() == {"detail": "Service is not available"}
 
     # Event without explicit argument, the client service from the request is not active
     response = APIClient().post(
@@ -430,7 +374,7 @@ def test_api_documents_search_access__request_inactive_services(settings):
     )
 
     assert response.status_code == 400
-    assert response.json() == {"detail": "Invalid request."}
+    assert response.json() == {"detail": "Service is not available"}
 
 
 @responses.activate
@@ -442,16 +386,10 @@ def test_api_documents_search_access__authenticated(settings):
     - only configured services providers are allowed (e.g docs)
     (groups is not yet implemnted)
     """
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
     setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs")
     token = build_authorization_bearer()
 
-    service = factories.ServiceFactory(client_id="docs")
+    service = factories.ServiceFactory(name="test-service", client_id="docs")
 
     documents_open = factories.DocumentSchemaFactory.build_batch(
         2, reach=enums.ReachEnum.PUBLIC
@@ -468,9 +406,7 @@ def test_api_documents_search_access__authenticated(settings):
     )
     documents = documents_user + documents_open + documents_restricted
 
-    prepare_index(
-        service.index_name, documents_user + documents_open + documents_restricted
-    )
+    prepare_index(service.name, documents_user + documents_open + documents_restricted)
 
     # Only owned documents (reach is ignored)
     response = APIClient().post(

src/backend/core/tests/test_embedding.py

@@ -1,125 +0,0 @@
-"""
-Test suite for opensearch embedding service
-"""
-
-import logging
-from json import dumps as json_dumps
-
-import pytest
-import responses
-
-from core.services.embedding import embed_text
-from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
-
-from .mock import albert_embedding_response
-from .utils import (
-    check_hybrid_search_enabled as check_hybrid_search_enabled_utils,
-)
-from .utils import (
-    enable_hybrid_search,
-)
-
-pytestmark = pytest.mark.django_db
-
-
-@pytest.fixture(autouse=True)
-def before_each():
-    """Clear caches before each test"""
-    clear_caches()
-    yield
-    clear_caches()
-
-
-def clear_caches():
-    """Clear caches used in opensearch service and factories"""
-    check_hybrid_search_enabled.cache_clear()
-    # the instance of check_hybrid_search_enabled used in utils.py
-    # is different and must be cleared separately
-    check_hybrid_search_enabled_utils.cache_clear()
-    opensearch_client().indices.delete(index="*")
-
-
-@responses.activate
-def test_embed_text_success(settings):
-    """Test embed_text retrieval is successful"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    text = "canine pet"
-
-    embedding = embed_text(text)
-
-    assert embedding == albert_embedding_response.response["data"][0]["embedding"]
-
-
-@responses.activate
-def test_embed_401_http_error(settings, caplog):
-    """Test embed_text does not crash and returns None on 401 error"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        status=401,
-        body=json_dumps({"message": "Authentication failed."}),
-    )
-    text = "canine pet"
-
-    with caplog.at_level(logging.WARNING):
-        embedding = embed_text(text)
-
-    assert any(
-        "embedding API request failed: 401 Client Error: Unauthorized" in message
-        for message in caplog.messages
-    )
-
-    assert embedding is None
-
-
-@responses.activate
-def test_embed_500_http_error(settings, caplog):
-    """Test embed_text does not crash and returns None on 500 error"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        status=500,
-        body=json_dumps({"message": "Internal server error."}),
-    )
-    text = "canine pet"
-
-    with caplog.at_level(logging.WARNING):
-        embedding = embed_text(text)
-
-    assert any(
-        "embedding API request failed: 500 Server Error: Internal Server Error"
-        in message
-        for message in caplog.messages
-    )
-
-    assert embedding is None
-
-
-@responses.activate
-def test_embed_wrong_format(settings, caplog):
-    """Test embed_text does not crash and returns None if api returns a wrong format"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json={"wrong": "format"},
-        status=200,
-    )
-    text = "canine pet"
-
-    with caplog.at_level(logging.WARNING):
-        embedding = embed_text(text)
-
-    assert any(
-        "unexpected embedding response format" in message for message in caplog.messages
-    )
-
-    assert embedding is None

src/backend/core/tests/test_indexing.py

@@ -1,227 +0,0 @@
-"""
-Test suite for opensearch indexing service
-"""
-
-import pytest
-import responses
-
-from core.services.indexing import detect_language_code, ensure_index_exists
-from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
-
-from .mock import albert_embedding_response
-from .utils import (
-    check_hybrid_search_enabled as check_hybrid_search_enabled_utils,
-)
-from .utils import (
-    enable_hybrid_search,
-)
-
-pytestmark = pytest.mark.django_db
-
-
-SERVICE_NAME = "test-service"
-
-
-@pytest.fixture(autouse=True)
-def before_each():
-    """Clear caches before each test"""
-    clear_caches()
-    yield
-    clear_caches()
-
-
-def clear_caches():
-    """Clear caches used in opensearch service and factories"""
-    check_hybrid_search_enabled.cache_clear()
-    # the instance of check_hybrid_search_enabled used in utils.py
-    # is different and must be cleared separately
-    check_hybrid_search_enabled_utils.cache_clear()
-    opensearch_client().indices.delete(index="*")
-
-
-@pytest.mark.parametrize(
-    "text, analyzer_name, expected_language_analyzer_tokens, expected_trigram_analyzer_tokens",
-    [
-        (
-            "l'éléphant a couru avec les Gens",
-            "french_analyzer",
-            # lowercase is applied ("Gens" -> "gens")
-            # asciifolding is applied ("éléphant" -> "elephant")
-            # stop words are removed ('avec', 'les')
-            # elisions are removed ("l'")
-            # stemming is applied ("gens" -> "gen")
-            ["elephant", "a", "couru", "gen"],
-            # lowercase is applied ("Gens" -> "gens")
-            # asciifolding is applied ("éléphant" -> "elephant")
-            # words smaller than 3 characters are removed ("a")
-            # trigrams are generated
-            [
-                "l'e",
-                "'el",
-                "ele",
-                "lep",
-                "eph",
-                "pha",
-                "han",
-                "ant",
-                "cou",
-                "our",
-                "uru",
-                "ave",
-                "vec",
-                "les",
-                "gen",
-                "ens",
-            ],
-        ),
-        (
-            "The Elephant is running into a café",
-            "english_analyzer",
-            # lowercase is applied ("Elephant" -> "elephant")
-            # asciifolding is applied ("café" -> "cafe")
-            # stop words are removed ("The", "into", "a")
-            # stemming is applied ("running" -> "run", "elephant" -> "eleph")
-            ["eleph", "run", "cafe"],
-            # lowercase is applied ("Gens" -> "gens")
-            # asciifolding is applied ("café" -> "cafe")
-            # trigrams are generated
-            # words smaller than 3 characters are removed ("a")
-            [
-                "the",
-                "ele",
-                "lep",
-                "eph",
-                "pha",
-                "han",
-                "ant",
-                "run",
-                "unn",
-                "nni",
-                "nin",
-                "ing",
-                "int",
-                "nto",
-                "caf",
-                "afe",
-            ],
-        ),
-        (
-            "Der Käfer läuft über die Straße",
-            "german_analyzer",
-            # lowercase is applied ("Der" -> "der", "Käfer" -> "käfer", "Straße" -> "straße")
-            # asciifolding is applied ("käfer" -> "kafer", "straße" -> "strass")
-            # stop words are removed ("Der", "die")
-            # stemming is applied ("kafer" -> "kaf")
-            ["kaf", "lauft", "uber", "strass"],
-            # lowercase is applied
-            # asciifolding is applied ("käfer" -> "kafer", "straße" -> "strasse")
-            # trigrams are generated
-            [
-                "der",
-                "kaf",
-                "afe",
-                "fer",
-                "lau",
-                "auf",
-                "uft",
-                "ube",
-                "ber",
-                "die",
-                "str",
-                "tra",
-                "ras",
-                "ass",
-                "sse",
-            ],
-        ),
-        (
-            "De Kinderen lopen naar de bakkerij",
-            "dutch_analyzer",
-            # lowercase is applied ("De" -> "de", "Kinderen" -> "kinderen")
-            # stop words are removed ("De", "naar", "de")
-            # stemming is applied ("kinderen" -> "kinder", "lopen" -> "lop")
-            ["kinder", "lop", "bakkerij"],
-            # lowercase is applied
-            # words smaller than 3 characters are removed ("de")
-            # trigrams are generated
-            [
-                "kin",
-                "ind",
-                "nde",
-                "der",
-                "ere",
-                "ren",
-                "lop",
-                "ope",
-                "pen",
-                "naa",
-                "aar",
-                "bak",
-                "akk",
-                "kke",
-                "ker",
-                "eri",
-                "rij",
-            ],
-        ),
-    ],
-)
-@responses.activate
-def test_opensearch_analyzers(
-    settings,
-    text,
-    analyzer_name,
-    expected_language_analyzer_tokens,
-    expected_trigram_analyzer_tokens,
-):
-    """Test the analyzers are correctly configured in OpenSearch"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    ensure_index_exists(SERVICE_NAME)
-
-    language_analyzer_response = opensearch_client().indices.analyze(
-        index=SERVICE_NAME,
-        body={
-            "analyzer": analyzer_name,
-            "text": text,
-        },
-    )
-    language_analyzer_tokens = [
-        token_info["token"] for token_info in language_analyzer_response["tokens"]
-    ]
-    response_trigram_analyzer = opensearch_client().indices.analyze(
-        index=SERVICE_NAME,
-        body={
-            "analyzer": "trigram_analyzer",
-            "text": text,
-        },
-    )
-    trigram_analyzer_tokens = [
-        token_info["token"] for token_info in response_trigram_analyzer["tokens"]
-    ]
-
-    assert expected_language_analyzer_tokens == language_analyzer_tokens
-    assert expected_trigram_analyzer_tokens == trigram_analyzer_tokens
-
-
-@pytest.mark.parametrize(
-    "text, expected_language_code",
-    [
-        ("This is a test sentence.", "en"),
-        ("Ceci est une phrase de test.", "fr"),
-        ("Dies ist ein Testsatz.", "de"),
-        ("Dit is een testzin.", "nl"),
-        ("Esta es una oración de prueba.", "und"),  # Spanish, unsupported
-        ("", "und"),
-        ("zefk,l", "und"),
-    ],
-)
-def test_detect_language_code(text, expected_language_code):
-    """Test detect_language_code function"""
-
-    assert detect_language_code(text) == expected_language_code

src/backend/core/tests/test_models_services.py

@@ -17,11 +17,10 @@ def test_models_services_name_unique():
         factories.ServiceFactory(name=service.name)
 
 
-def test_models_services_name_slugified(settings):
+def test_models_services_name_slugified():
     """The name field should be slugified."""
     service = factories.ServiceFactory(name="My service name")
     assert service.name == "my-service-name"
-    assert service.index_name == f"{settings.OPENSEARCH_INDEX_PREFIX}-my-service-name"
 
 
 def test_models_services_token_50_characters_exact():

src/backend/core/tests/test_search.py

@@ -1,714 +0,0 @@
-"""
-Test suite for opensearch search service
-"""
-
-import logging
-import operator
-from json import dumps as json_dumps
-
-import pytest
-import responses
-
-from core import enums, factories
-from core.services import opensearch
-from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
-from core.services.search import search
-from core.utils import bulk_create_documents, delete_search_pipeline, prepare_index
-
-from .mock import albert_embedding_response
-from .utils import (
-    check_hybrid_search_enabled as check_hybrid_search_enabled_utils,
-)
-from .utils import (
-    enable_hybrid_search,
-)
-
-pytestmark = pytest.mark.django_db
-
-SERVICE_NAME = "test-service"
-
-
-def search_params(service):
-    """Build opensearch.search() parameters for tests using the service index name"""
-    return {
-        "nb_results": 20,
-        "order_by": "relevance",
-        "order_direction": "desc",
-        "search_indices": {service.index_name},
-        "reach": None,
-        "user_sub": "user_sub",
-        "groups": [],
-        "visited": [],
-        "tags": [],
-        "search_type": enums.SearchTypeEnum.HYBRID,
-    }
-
-
-@pytest.fixture(autouse=True)
-def before_each():
-    """Clear caches and delete search pipeline before each test"""
-    clear_caches()
-    yield
-    clear_caches()
-
-
-def clear_caches():
-    """Clear caches used in opensearch service and factories"""
-    check_hybrid_search_enabled.cache_clear()
-    # the instance of check_hybrid_search_enabled used in utils.py
-    # is different and must be cleared separately
-    check_hybrid_search_enabled_utils.cache_clear()
-    delete_search_pipeline()
-    opensearch_client().indices.delete(index="*")
-
-
-@responses.activate
-def test_hybrid_search_success(settings, caplog):
-    """Test the hybrid search is successful"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "canine pet"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        f"Performing hybrid search with embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    assert result["hits"]["max_score"] > 0.0
-    # hybrid search always returns a response of fixed sized sorted and scored by relevance
-    assert {hit["_source"]["title.en"] for hit in result["hits"]["hits"]} == {
-        doc["title"] for doc in documents
-    }
-
-
-@responses.activate
-def test_hybrid_search_without_embedded_index(settings, caplog):
-    """Test the hybrid search is successful"""
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves"},
-            {"title": "dog", "content": "dogs"},
-            {"title": "cat", "content": "cats"},
-        ]
-    )
-    # index is prepared but hybrid search is not yet enable.
-    # they then won't be embedded.
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    # check embedding is None
-    indexed_documents = opensearch.opensearch_client().search(
-        index=service.index_name, body={"query": {"match_all": {}}}
-    )
-    assert indexed_documents["hits"]["hits"][0]["_source"]["chunks"] is None
-
-    # hybrid search is enabled before to do the first requests
-    enable_hybrid_search(settings)
-
-    q = "canine pet"
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    # the hybrid search is done successfully
-    assert any(
-        f"Performing hybrid search with embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    # but no match can obviously be found
-    assert result["hits"]["max_score"] == 0.0
-    assert len(result["hits"]["hits"]) == 0
-
-    # The full-text search is still functional
-    q = "wolf"
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        f"Performing hybrid search with embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert (
-        result["hits"]["hits"][0]["_source"][
-            f"title.{settings.UNDETERMINED_LANGUAGE_CODE}"
-        ]
-        == q
-    )
-
-
-def test_fall_back_on_full_text_search_if_hybrid_search_disabled(settings, caplog):
-    """Test the full-text search is done when HYBRID_SEARCH_ENABLED=False"""
-    enable_hybrid_search(settings)
-    settings.HYBRID_SEARCH_ENABLED = False
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "wolf"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        "Hybrid search is disabled via HYBRID_SEARCH_ENABLED setting" in message
-        for message in caplog.messages
-    )
-    assert any(
-        f"Performing full-text search without embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert result["hits"]["hits"][0]["_source"]["title.en"] == "wolf"
-
-
-@responses.activate
-def test_force_full_text_search_with_search_type_parameter(settings, caplog):
-    """Test the full-text search is done when search_type=FULL_TEXT even if hybrid is enabled"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "wolf"
-    with caplog.at_level(logging.INFO):
-        result = search(
-            q=q,
-            **{**search_params(service), "search_type": enums.SearchTypeEnum.FULL_TEXT},
-        )
-
-    assert any(
-        f"Performing full-text search without embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert result["hits"]["hits"][0]["_source"]["title.en"] == "wolf"
-
-
-def test_request_hybrid_search_when_server_has_it_disabled(settings, caplog):
-    """Test warning when hybrid search is requested but disabled on server"""
-    enable_hybrid_search(settings)
-    settings.HYBRID_SEARCH_ENABLED = False
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "canine pet"
-    with caplog.at_level(logging.INFO):
-        search(
-            q=q,
-            **{**search_params(service), "search_type": enums.SearchTypeEnum.HYBRID},
-        )
-
-    assert any(
-        "Hybrid search was requested (search_type=hybrid) but is disabled on server"
-        in message
-        for message in caplog.messages
-    )
-    assert any(
-        f"Performing full-text search without embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-
-@responses.activate
-def test_api_documents_search_with_search_type_hybrid(settings, caplog):
-    """Test API with search_type=hybrid uses hybrid search when enabled"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-        ]
-    )
-    prepare_index(service.index_name, documents)
-
-    q = "canine pet"
-    with caplog.at_level(logging.INFO):
-        result = search(
-            q=q,
-            **{**search_params(service), "search_type": enums.SearchTypeEnum.HYBRID},
-        )
-
-    assert any(
-        f"Performing hybrid search with embedding: {q}" in message
-        for message in caplog.messages
-    )
-
-    assert result["hits"]["max_score"] > 0.0
-    # hybrid search always returns a response of fixed sized sorted and scored by relevance
-    assert {hit["_source"]["title.en"] for hit in result["hits"]["hits"]} == {
-        doc["title"] for doc in documents
-    }
-
-
-@responses.activate
-def test_fall_back_on_full_text_search_if_embedding_api_fails(settings, caplog):
-    """Test the full-text search is done when the embedding api fails"""
-
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        status=401,
-        body=json_dumps({"message": "Authentication failed."}),
-    )
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "wolf"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        "embedding API request failed: 401 Client Error: Unauthorized" in message
-        for message in caplog.messages
-    )
-    assert any(
-        f"Performing full-text search without embedding: {q}" in message
-        for message in caplog.messages
-    )
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert result["hits"]["hits"][0]["_source"]["title.en"] == "wolf"
-
-
-@responses.activate
-def test_fall_back_on_full_text_search_if_variable_are_missing(settings, caplog):
-    """Test the full-text search is done when variables are missing for hybrid search"""
-    enable_hybrid_search(settings)
-    del settings.HYBRID_SEARCH_WEIGHTS
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "wolf"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any(
-        "Missing variables for hybrid search: HYBRID_SEARCH_WEIGHTS" in message
-        for message in caplog.messages
-    )
-    assert any(
-        f"Performing full-text search without embedding: {q}" in message
-        for message in caplog.messages
-    )
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 1
-    assert result["hits"]["hits"][0]["_source"]["title.en"] == "wolf"
-
-
-@responses.activate
-def test_match_all(settings, caplog):
-    """Test match all when q='*' and no semantic search is needed"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "*"
-    with caplog.at_level(logging.INFO):
-        result = search(q=q, **search_params(service))
-
-    assert any("Performing match_all query" in message for message in caplog.messages)
-    assert result["hits"]["max_score"] > 0.0
-    assert len(result["hits"]["hits"]) == 3
-
-
-@responses.activate
-def test_search_ordering_by_relevance(settings, caplog):
-    """Test the hybrid supports ordering by relevance asc and desc"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    q = "canine pet"
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    for direction in ["asc", "desc"]:
-        with caplog.at_level(logging.INFO):
-            result = search(
-                q=q, **{**search_params(service), "order_direction": direction}
-            )
-
-        # Check that results are sorted by score as expected
-        hits = result["hits"]["hits"]
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(hits) - 1):
-            assert compare(hits[i]["_score"], hits[i + 1]["_score"])
-
-
-@responses.activate
-def test_hybrid_search_number_of_matches(settings):
-    """
-    In this test full-text search always return 0 documents.
-    The test checks the number of hits returned by hybrid search with different k values.
-    """
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    q = "pony"  # full-text matches 0 document
-    for nb_results in [1, 2, 3]:  # semantic should match k documents
-        result = search(q=q, **{**search_params(service), "nb_results": nb_results})
-        assert len(result["hits"]["hits"]) == nb_results
-
-
-def test_search_filtering_by_single_tag():
-    """Test filtering documents by a single tag"""
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-
-    documents_to_search = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["tag-to-search"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["tag-to-search", "tag-to-filter"]
-        ),
-    ]
-    documents_to_filter = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["tag-to-filter"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-        ),
-    ]
-    expected_ids = {str(doc["id"]) for doc in documents_to_search}
-
-    prepare_index(service.index_name, documents_to_search + documents_to_filter)
-
-    # Search for documents with tag-to-search tag
-    result = search(q="*", **{**search_params(service), "tags": ["tag-to-search"]})
-    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
-
-    assert result["hits"]["total"]["value"] == len(documents_to_search)
-    assert returned_ids == expected_ids
-
-
-def test_search_filtering_by_multiple_tags():
-    """Test filtering documents by multiple tags (OR logic)"""
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-
-    documents_to_search = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["tag-to-search-1"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["tag-to-search-1", "tag-to-filter"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["tag-to-search-2", "tag-to-filter"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["tag-to-search-1", "tag-to-search-2"]
-        ),
-    ]
-    documents_to_filter = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], tags=["tag-to-filter"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-        ),
-    ]
-    expected_ids = {str(doc["id"]) for doc in documents_to_search}
-
-    prepare_index(service.index_name, documents_to_search + documents_to_filter)
-
-    # Search for documents with tag-to-search-1 OR tag-to-search-2 tags
-    result = search(
-        q="*",
-        **{**search_params(service), "tags": ["tag-to-search-1", "tag-to-search-2"]},
-    )
-    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
-
-    assert result["hits"]["total"]["value"] == len(documents_to_search)
-    assert returned_ids == expected_ids
-
-
-def test_search_no_tags_filter_returns_all():
-    """Test that not providing tags filter returns all documents"""
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-
-    documents = bulk_create_documents(
-        [
-            {
-                "title": "Document without tags",
-                "content": "Untagged document",
-            },
-            {
-                "title": "Document with tags",
-                "content": "Tagged document",
-                "tags": ["tag-to-search"],
-            },
-        ]
-    )
-    prepare_index(service.index_name, documents)
-
-    # Search without tags filter
-    result = search(q="*", **search_params(service))
-
-    assert result["hits"]["total"]["value"] == len(documents)
-
-
-def test_search_filtering_by_tag_and_query():
-    """Test filtering documents by both tag and query text"""
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-
-    documents_to_search = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], title="title to search", tags=["tag-to-search"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to search 1",
-            tags=["tag-to-search", "tag-to-filter"],
-        ),
-    ]
-    documents_to_filter = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], title="title to filter", tags=["tag-to-filter"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], title="title to filter 1", tags=["tag-to-filter"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to search 2",
-        ),
-    ]
-    expected_ids = {str(doc["id"]) for doc in documents_to_search}
-
-    prepare_index(service.index_name, documents_to_search + documents_to_filter)
-
-    # Search with both query and tag filter
-    result = search(q="search", **{**search_params(service), "tags": ["tag-to-search"]})
-    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
-
-    assert result["hits"]["total"]["value"] == len(documents_to_search)
-    assert returned_ids == expected_ids
-
-
-def test_search_filtering_by_path():
-    """Test filtering documents by path prefix"""
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-
-    documents_to_search = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], path="path/to/search/doc-1"
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], path="path/to/search/doc-2"
-        ),
-    ]
-    documents_to_filter = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], path="path/to/filter/doc-3"
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-        ),
-    ]
-    expected_ids = {str(doc["id"]) for doc in documents_to_search}
-
-    prepare_index(service.index_name, documents_to_search + documents_to_filter)
-
-    path_filter = "path/to/search"
-    result = search(q="*", **{**search_params(service), "path": path_filter})
-    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
-
-    assert result["hits"]["total"]["value"] == len(documents_to_search)
-    assert returned_ids == expected_ids
-
-
-def test_search_filtering_by_query_path_and_tag():
-    """Test filtering documents by query text, path prefix and tag combined"""
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-
-    documents_to_search = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to search 0",
-            path="path/to/search-0",
-            tags=["tag-to-search"],
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to search 1",
-            path="path/to/search/doc1",
-            tags=["tag-to-search", "tag-to-filter"],
-        ),
-    ]
-    documents_to_filter = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to filter",
-            path="path/to/search/doc-3",
-            tags=["tag-to-search"],
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to search 4",
-            path="path/to/filter/doc-4",
-            tags=["tag-to-search"],
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to search 4",
-            path="path/to/search/doc-4",
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"], title="title to search 5", tags=["tag-to-search"]
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to search 6",
-            path="path/to/search/doc-6",
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="title to search 7",
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="",
-            path="path/to/search/doc-8",
-            tags=["tag-to-search"],
-        ),
-    ]
-    expected_ids = {str(doc["id"]) for doc in documents_to_search}
-
-    prepare_index(service.index_name, documents_to_search + documents_to_filter)
-
-    # Search with query, path and tag filters combined
-    result = search(
-        q="search",
-        **{
-            **search_params(service),
-            "path": "path/to/search",
-            "tags": ["tag-to-search"],
-        },
-    )
-    returned_ids = {hit["_id"] for hit in result["hits"]["hits"]}
-
-    assert result["hits"]["total"]["value"] == len(documents_to_search)
-    assert returned_ids == expected_ids

src/backend/core/tests/test_selftests.py

@@ -1,145 +0,0 @@
-"""Tests for the selftest system."""
-
-from core.selftests import SelfTest, SelfTestRegistry, SelfTestResult
-
-
-class DummySuccessTest(SelfTest):
-    """A test that always succeeds."""
-
-    name = "Dummy Success Test"
-    description = "A test that always passes"
-
-    def run(self) -> SelfTestResult:
-        return SelfTestResult(
-            name=self.name,
-            success=True,
-            message="Test passed successfully",
-            duration_ms=10.0,
-        )
-
-
-class DummyFailureTest(SelfTest):
-    """A test that always fails."""
-
-    name = "Dummy Failure Test"
-    description = "A test that always fails"
-
-    def run(self) -> SelfTestResult:
-        return SelfTestResult(
-            name=self.name,
-            success=False,
-            message="Test failed as expected",
-            duration_ms=5.0,
-        )
-
-
-class DummyExceptionTest(SelfTest):
-    """A test that raises an exception."""
-
-    name = "Dummy Exception Test"
-    description = "A test that throws an exception"
-
-    def run(self) -> SelfTestResult:
-        raise RuntimeError("This test is designed to fail")
-
-
-def test_register_test():
-    """Test that a test can be registered."""
-    registry = SelfTestRegistry()
-    registry.register(DummySuccessTest)
-
-    tests = registry.get_all_tests()
-
-    assert len(tests) == 1
-    assert isinstance(tests[0], DummySuccessTest)
-
-
-def test_register_multiple_tests():
-    """Test that multiple tests can be registered."""
-    registry = SelfTestRegistry()
-    registry.register(DummySuccessTest)
-    registry.register(DummyFailureTest)
-
-    tests = registry.get_all_tests()
-
-    assert len(tests) == 2
-
-
-def test_unregister_test():
-    """Test that a test can be unregistered."""
-    registry = SelfTestRegistry()
-    registry.register(DummySuccessTest)
-    registry.unregister(DummySuccessTest)
-
-    tests = registry.get_all_tests()
-
-    assert len(tests) == 0
-
-
-def test_run_all_success():
-    """Test running all tests when all pass."""
-    registry = SelfTestRegistry()
-    registry.register(DummySuccessTest)
-
-    results = registry.run_all()
-
-    assert len(results) == 1
-    assert results[0].success is True
-
-
-def test_run_all_mixed():
-    """Test running all tests with mixed results."""
-    registry = SelfTestRegistry()
-    registry.register(DummySuccessTest)
-    registry.register(DummyFailureTest)
-
-    results = registry.run_all()
-
-    assert len(results) == 2
-    assert results[0].success is True
-    assert results[1].success is False
-
-
-def test_run_all_with_exception():
-    """Test that exceptions are caught and converted to failures."""
-    registry = SelfTestRegistry()
-    registry.register(DummyExceptionTest)
-
-    results = registry.run_all()
-
-    assert len(results) == 1
-    assert results[0].success is False
-    assert "exception" in results[0].message.lower()
-
-
-def test_result_to_dict():
-    """Test converting a result to a dictionary."""
-    result = SelfTestResult(
-        name="Test Name",
-        success=True,
-        message="Test message",
-        details={"key": "value"},
-        duration_ms=100.5,
-    )
-
-    result_dict = result.to_dict()
-
-    assert result_dict["name"] == "Test Name"
-    assert result_dict["success"] is True
-    assert result_dict["message"] == "Test message"
-    assert result_dict["details"]["key"] == "value"
-    assert result_dict["duration_ms"] == 100.5
-
-
-def test_result_without_optional_fields():
-    """Test creating a result without optional fields."""
-    result = SelfTestResult(
-        name="Test Name",
-        success=False,
-        message="Test failed",
-    )
-
-    result_dict = result.to_dict()
-
-    assert result_dict["details"] == {}
-    assert result_dict["duration_ms"] is None

src/backend/core/tests/utils.py

@@ -1,33 +1,50 @@
-"""Utility functions for Test."""
+"""Tests Service model for find's core app."""
 
 import base64
 import json
-import logging
 from functools import partial
+from typing import List
 
-from core.management.commands.create_search_pipeline import (
-    ensure_search_pipeline_exists,
-)
-from core.services.opensearch import (
-    check_hybrid_search_enabled,
-)
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from joserfc import jwe as jose_jwe
+from joserfc import jwt as jose_jwt
+from joserfc.jwk import RSAKey
+from jwt.utils import to_base64url_uint
+from opensearchpy.helpers import bulk
 
-logger = logging.getLogger(__name__)
+from core import opensearch
 
 
-def enable_hybrid_search(settings):
-    """Enable hybrid search settings for tests."""
-    settings.HYBRID_SEARCH_ENABLED = True
-    settings.HYBRID_SEARCH_WEIGHTS = [0.3, 0.7]
-    settings.EMBEDDING_API_KEY = "test-api-key"
-    settings.EMBEDDING_API_PATH = "https://test.embedding.api/v1/embeddings"
-    settings.EMBEDDING_REQUEST_TIMEOUT = 10
-    settings.EMBEDDING_API_MODEL_NAME = "embeddings-small"
-    settings.EMBEDDING_DIMENSION = 1024
+def delete_test_indices():
+    """Drop all search index containing the 'test' word"""
+    opensearch.client.indices.delete(index="*test*")
 
-    # Clear the cache here or the hybrid search will remain disabled
-    check_hybrid_search_enabled.cache_clear()
-    ensure_search_pipeline_exists()
+
+def prepare_index(index_name, documents: List, cleanup=True):
+    """Prepare the search index before testing a query on it."""
+    if cleanup:
+        delete_test_indices()
+
+    opensearch.ensure_index_exists(index_name)
+
+    # Index new documents
+    actions = [
+        {
+            "_op_type": "index",
+            "_index": index_name,
+            "_id": doc["id"],
+            "_source": {k: v for k, v in doc.items() if k != "id"},
+        }
+        for doc in documents
+    ]
+    bulk(opensearch.client, actions)
+
+    # Force refresh again so all changes are visible to search
+    opensearch.client.indices.refresh(index=index_name)
+
+    count = opensearch.client.count(index=index_name)["count"]
+    assert count == len(documents), f"Expected {len(documents)}, got {count}"
 
 
 def build_authorization_bearer(token="some_token"):
@@ -43,6 +60,116 @@ def build_authorization_bearer(token="some_token"):
     return base64.b64encode(token.encode("utf-8")).decode("utf-8")
 
 
+def setup_oicd_jwt_resource_server(
+    responses,
+    settings,
+    sub="some_sub",
+    audience="some_client_id",
+):
+    """
+    Setup settings for a resource server with JWT backend.
+    Simulate an encrypted token introspection.
+    NOTE : Use it with @responses.activate or the fake introspection view will not work.
+    """
+    token_data = {
+        "sub": sub,
+        "iss": "https://oidc.example.com",
+        "aud": audience,
+        "client_id": "some_service_provider",
+        "scope": "docs",
+        "active": True,
+    }
+
+    private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+
+    unencrypted_pem_private_key = private_key.private_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PrivateFormat.TraditionalOpenSSL,
+        encryption_algorithm=serialization.NoEncryption(),
+    )
+
+    pem_public_key = private_key.public_key().public_bytes(
+        encoding=serialization.Encoding.PEM,
+        format=serialization.PublicFormat.SubjectPublicKeyInfo,
+    )
+
+    settings.OIDC_RS_PRIVATE_KEY_STR = unencrypted_pem_private_key.decode("utf-8")
+    settings.OIDC_RS_ENCRYPTION_KEY_TYPE = "RSA"
+    settings.OIDC_RS_ENCRYPTION_ENCODING = "A256GCM"
+    settings.OIDC_RS_ENCRYPTION_ALGO = "RSA-OAEP"
+    settings.OIDC_RS_SIGNING_ALGO = "RS256"
+    settings.OIDC_RS_CLIENT_ID = audience
+    settings.OIDC_RS_CLIENT_SECRET = "some_client_secret"
+    settings.OIDC_RS_SCOPES = ["openid", "docs", "email"]
+
+    settings.OIDC_OP_URL = "https://oidc.example.com"
+    settings.OIDC_OP_JWKS_ENDPOINT = "https://oidc.example.com/jwks"
+    settings.OIDC_OP_INTROSPECTION_ENDPOINT = "https://oidc.example.com/introspect"
+
+    settings.OIDC_VERIFY_SSL = False
+    settings.OIDC_TIMEOUT = 5
+    settings.OIDC_PROXY = None
+    settings.OIDC_CREATE_USER = False
+
+    # Mock the JWKS endpoint
+    public_numbers = private_key.public_key().public_numbers()
+    responses.add(
+        responses.GET,
+        settings.OIDC_OP_JWKS_ENDPOINT,
+        body=json.dumps(
+            {
+                "keys": [
+                    {
+                        "kty": settings.OIDC_RS_ENCRYPTION_KEY_TYPE,
+                        "alg": settings.OIDC_RS_SIGNING_ALGO,
+                        "use": "sig",
+                        "kid": "1234567890",
+                        "n": to_base64url_uint(public_numbers.n).decode("ascii"),
+                        "e": to_base64url_uint(public_numbers.e).decode("ascii"),
+                    }
+                ]
+            }
+        ),
+    )
+
+    def encrypt_jwt(json_data):
+        """Encrypt the JWT token for the backend to decrypt."""
+        token = jose_jwt.encode(
+            {
+                "kid": "1234567890",
+                "alg": settings.OIDC_RS_SIGNING_ALGO,
+            },
+            json_data,
+            RSAKey.import_key(unencrypted_pem_private_key),
+            algorithms=[settings.OIDC_RS_SIGNING_ALGO],
+        )
+
+        return jose_jwe.encrypt_compact(
+            protected={
+                "alg": settings.OIDC_RS_ENCRYPTION_ALGO,
+                "enc": settings.OIDC_RS_ENCRYPTION_ENCODING,
+            },
+            plaintext=token,
+            public_key=RSAKey.import_key(pem_public_key),
+            algorithms=[
+                settings.OIDC_RS_ENCRYPTION_ALGO,
+                settings.OIDC_RS_ENCRYPTION_ENCODING,
+            ],
+        )
+
+    responses.add(
+        responses.POST,
+        "https://oidc.example.com/introspect",
+        body=encrypt_jwt(
+            {
+                "iss": "https://oidc.example.com",
+                "aud": audience,  # settings.OIDC_RS_CLIENT_ID
+                "token_introspection": token_data,
+            }
+        ),
+    )
+
+
 def setup_oicd_resource_server(
     responses,
     settings,
@@ -83,12 +210,12 @@ def setup_oicd_resource_server(
     if callable(introspect):
         responses.add_callback(
             responses.POST,
-            settings.OIDC_OP_INTROSPECTION_ENDPOINT,
+            "https://oidc.example.com/introspect",
             callback=partial(introspect, user_info=token_data),
         )
     else:
         responses.add(
             responses.POST,
-            settings.OIDC_OP_INTROSPECTION_ENDPOINT,
+            "https://oidc.example.com/introspect",
             body=json.dumps(token_data),
         )

src/backend/core/urls.py

@@ -2,11 +2,10 @@
 
 from django.urls import include, path
 
-from .views import DeleteDocumentsView, IndexDocumentView, SearchDocumentView
+from .views import IndexDocumentView, SearchDocumentView
 
 urlpatterns = [
     path("documents/index/", IndexDocumentView.as_view(), name="document"),
     path("documents/search/", SearchDocumentView.as_view(), name="document"),
-    path("documents/delete/", DeleteDocumentsView.as_view(), name="document"),
     path("", include("lasuite.oidc_resource_server.urls")),
 ]

src/backend/core/utils.py

@@ -1,80 +0,0 @@
-"""Tests Service model for find's core app."""
-
-import logging
-from typing import List
-
-from django.conf import settings as django_settings
-
-from opensearchpy.exceptions import NotFoundError
-from opensearchpy.helpers import bulk
-
-from core import factories
-from core.services.indexing import ensure_index_exists, prepare_document_for_indexing
-from core.services.opensearch import opensearch_client
-
-logger = logging.getLogger(__name__)
-
-
-def bulk_create_documents(document_payloads):
-    """Create documents in bulk from payloads"""
-    return [
-        factories.DocumentSchemaFactory.build(**document_payload, users=["user_sub"])
-        for document_payload in document_payloads
-    ]
-
-
-def delete_search_pipeline():
-    """Delete the hybrid search pipeline if it exists"""
-    logger.info(
-        "Deleting search pipeline %s", django_settings.HYBRID_SEARCH_PIPELINE_ID
-    )
-
-    try:
-        opensearch_client().transport.perform_request(
-            method="DELETE",
-            url=f"/_search/pipeline/{django_settings.HYBRID_SEARCH_PIPELINE_ID}",
-        )
-    except NotFoundError:
-        logger.info("Search pipeline not found, nothing to delete.")
-
-
-def delete_index(index_name):
-    """Delete the hybrid search pipeline if it exists"""
-    logger.info("Deleting Index %s", index_name)
-
-    try:
-        opensearch_client().indices.delete(index=index_name)
-    except NotFoundError:
-        logger.info("Search pipeline %s not found, nothing to delete.", index_name)
-
-
-def prepare_index(index_name, documents: List):
-    """Prepare the search index before testing a query on it."""
-    logger.info("Preparing index %s with %d documents", index_name, len(documents))
-
-    ensure_index_exists(index_name)
-    actions = [
-        {
-            "_op_type": "index",
-            "_index": index_name,
-            "_id": document["id"],
-            "_source": prepare_document_for_indexing(document),
-        }
-        for document in documents
-    ]
-    bulk(opensearch_client(), actions)
-    opensearch_client().indices.refresh(index=index_name)
-
-
-def get_language_value(source, language_field):
-    """
-    extract the value of the language field with the correct language_code extension.
-    "title" and "content" have extensions like "title.en" or "title.fr".
-    get_language_value will return the value regardless of the extension.
-    """
-    for language_code in django_settings.SUPPORTED_LANGUAGE_CODES:
-        if f"{language_field}.{language_code}" in source:
-            return source[f"{language_field}.{language_code}"]
-    raise ValueError(
-        f"No '{language_field}' field with any supported language code in object"
-    )

src/backend/core/views.py

@@ -10,18 +10,11 @@ from pydantic import ValidationError as PydanticValidationError
 from rest_framework import status, views
 from rest_framework.response import Response
 
-from . import schemas
+from . import enums, schemas
 from .authentication import ServiceTokenAuthentication
-from .enums import SearchTypeEnum
+from .models import Service
+from .opensearch import client, ensure_index_exists
 from .permissions import IsAuthAuthenticated
-from .services.indexing import (
-    ensure_index_exists,
-    get_opensearch_indices,
-    prepare_document_for_indexing,
-)
-from .services.opensearch import check_hybrid_search_enabled, opensearch_client
-from .services.search import search
-from .utils import get_language_value
 
 logger = logging.getLogger(__name__)
 
@@ -37,6 +30,7 @@ class IndexDocumentView(views.APIView):
     authentication_classes = [ServiceTokenAuthentication]
     permission_classes = [IsAuthAuthenticated]
 
+    # pylint: disable=too-many-locals
     def post(self, request, *args, **kwargs):
         """
         API view for indexing documents into OpenSearch index of the authenticated service.
@@ -85,218 +79,63 @@ class IndexDocumentView(views.APIView):
             - Returns a list of results for all documents, with details of success and indexing
               errors.
         """
-        index_name = request.auth.index_name
-        opensearch_client_ = opensearch_client()
+        index_name = request.auth.name
 
         if isinstance(request.data, list):
-            return self.bulk_index(request, index_name, opensearch_client_)
+            # Bulk indexing several documents
+            results = []
+            actions = []
+            has_errors = False
 
-        return self.single_index(request, index_name, opensearch_client_)
+            for i, document_data in enumerate(request.data):
+                try:
+                    document = schemas.DocumentSchema(**document_data)
+                except PydanticValidationError as excpt:
+                    errors = [
+                        {key: error[key] for key in ("msg", "type", "loc")}
+                        for error in excpt.errors()
+                    ]
+                    results.append({"index": i, "status": "error", "errors": errors})
+                    has_errors = True
+                else:
+                    document_dict = document.model_dump()
+                    _id = document_dict.pop("id")
+                    actions.append({"index": {"_id": _id}})
+                    actions.append(document_dict)
+                    results.append({"index": i, "_id": _id, "status": "valid"})
 
-    def single_index(self, request, index_name, opensearch_client_):
-        """
-        Index a single document into OpenSearch.
+            if has_errors:
+                return Response(results, status=status.HTTP_400_BAD_REQUEST)
 
-        Args:
-            request: The HTTP request containing document data.
-            index_name: The name of the OpenSearch index.
-            opensearch_client_: The OpenSearch client instance.
+            # Build index if needed.
+            ensure_index_exists(index_name)
 
-        Returns:
-            Response: HTTP response with status and document ID.
-                - 201 Created: Returns the indexed document ID.
-                - 400 Bad Request: Returns an error message if the document is invalid.
-        """
-        document_dict = prepare_document_for_indexing(
-            schemas.DocumentSchema(**request.data).model_dump()
-        )
+            response = client.bulk(index=index_name, body=actions)
+            for i, item in enumerate(response["items"]):
+                if item["index"]["status"] != 201:
+                    results[i]["status"] = "error"
+                    results[i]["message"] = (
+                        item["index"].get("error", {}).get("reason", "Unknown error")
+                    )
+                else:
+                    results[i]["status"] = "success"
+
+            return Response(results, status=status.HTTP_207_MULTI_STATUS)
+
+        # Indexing a single document
+        document = schemas.DocumentSchema(**request.data)
+        document_dict = document.model_dump()
         _id = document_dict.pop("id")
-        logger.info(
-            "Indexing single document %s on index %s",
-            get_language_value(document_dict, "title"),
-            index_name,
-        )
 
+        # Build index if needed.
         ensure_index_exists(index_name)
-        opensearch_client_.index(
-            index=index_name,
-            body=document_dict,
-            id=_id,
-        )
+
+        client.index(index=index_name, body=document_dict, id=_id)
 
         return Response(
             {"status": "created", "_id": _id}, status=status.HTTP_201_CREATED
         )
 
-    # pylint: disable=too-many-locals
-    def bulk_index(self, request, index_name, opensearch_client_):
-        """
-        Index multiple documents into OpenSearch in bulk.
-
-        Args:
-            request: The HTTP request containing a list of documents.
-            index_name: The name of the OpenSearch index.
-            opensearch_client_: The OpenSearch client instance.
-
-        Returns:
-            Response: HTTP response with detailed status for each document.
-                - 201 Created: Returns status for all documents.
-                - 400 Bad Request: Returns errors if document validation fails.
-        """
-        results = []
-        actions = []
-        has_errors = False
-
-        for i, document_data in enumerate(request.data):
-            try:
-                document = schemas.DocumentSchema(**document_data)
-            except PydanticValidationError as excpt:
-                errors = [
-                    {key: error[key] for key in ("msg", "type", "loc")}
-                    for error in excpt.errors()
-                ]
-                results.append({"index": i, "status": "error", "errors": errors})
-                has_errors = True
-            else:
-                document_dict = prepare_document_for_indexing(document.model_dump())
-                logger.info(
-                    "Indexing document %s on index %s",
-                    get_language_value(document_dict, "title"),
-                    index_name,
-                )
-                _id = document_dict.pop("id")
-                actions.append({"index": {"_id": _id}})
-                actions.append(document_dict)
-                results.append({"index": i, "_id": _id, "status": "valid"})
-
-        if has_errors:
-            return Response(results, status=status.HTTP_400_BAD_REQUEST)
-
-        ensure_index_exists(index_name)
-        response = opensearch_client_.bulk(index=index_name, body=actions)
-        for i, item in enumerate(response["items"]):
-            if item["index"]["status"] != 201:
-                results[i]["status"] = "error"
-                results[i]["message"] = (
-                    item["index"].get("error", {}).get("reason", "Unknown error")
-                )
-            else:
-                results[i]["status"] = "success"
-
-        return Response(results, status=status.HTTP_201_CREATED)
-
-
-class DeleteDocumentsView(ResourceServerMixin, views.APIView):
-    """
-    API view for deleting documents from OpenSearch.
-        - Allows authenticated users to delete documents from a specified index.
-        - Users can only delete documents where they are listed in the 'users' field.
-        - Returns the count of deleted documents without revealing document existence.
-    """
-
-    authentication_classes = [ResourceServerAuthentication]
-    permission_classes = [IsAuthAuthenticated]
-
-    def post(self, request, *args, **kwargs):
-        """
-        Handle POST requests to delete documents from the specified index.
-
-        Only documents where the authenticated user is in the 'users' field will be deleted.
-
-        Body Parameters:
-        ---------------
-        service: str
-            service name to determine the index from which to delete documents.
-        document_ids : List[str], optional
-            A list of document IDs to delete from the index.
-        tags : List[str], optional
-            A list of tags to filter documents for deletion.
-
-        At least one of document_ids or tags must be provided.
-        The list of ids and the list of tags are combined with AND logic.
-
-        Returns:
-        --------
-        Response : rest_framework.response.Response
-            - 200 OK: returns a JSON object with the following keys:
-                - nb-deleted-documents: Number of documents deleted.
-                - undeleted-document-ids: sublist of param.document_ids that were not deleted.
-                Deletion may be prevented because the document does not exist,
-                because the user is not authorized to delete it or because a tag filter was used.
-            - 400 Bad Request: If parameters are invalid or missing.
-        """
-        params = schemas.DeleteDocumentsSchema(**request.data)
-        try:
-            index_name = get_opensearch_indices(
-                self._get_service_provider_audience(), services=[params.service]
-            )[0]
-        except SuspiciousOperation as e:
-            logger.error(e)
-            return Response(
-                {"detail": "Invalid request."},
-                status=status.HTTP_400_BAD_REQUEST,
-            )
-
-        logger.info(
-            "Deleting documents from index %s with filters: document_ids=%s, tags=%s",
-            index_name,
-            params.document_ids,
-            params.tags,
-        )
-
-        client = opensearch_client()
-        deletable_matches = client.search(
-            index=index_name,
-            body={
-                "query": self._build_query(
-                    self.request.user.sub,
-                    document_ids=params.document_ids,
-                    tags=params.tags,
-                )
-            },
-        )
-        deletable_ids = [hit["_id"] for hit in deletable_matches["hits"]["hits"]]
-
-        if deletable_ids:
-            response = client.delete_by_query(
-                index=index_name,
-                body={"query": {"ids": {"values": deletable_ids}}},
-            )
-            nb_deleted = response.get("deleted", 0)
-        else:
-            nb_deleted = 0
-
-        return Response(
-            {
-                "nb-deleted-documents": nb_deleted,
-                "undeleted-document-ids": [
-                    document_id
-                    for document_id in params.document_ids or []
-                    if document_id not in deletable_ids
-                ],
-            },
-            status=status.HTTP_200_OK,
-        )
-
-    def _build_query(self, user_sub, document_ids=None, tags=None):
-        """
-        Build OpenSearch query for document deletion.
-
-        Args:
-            user_sub: User subject identifier for authorization.
-            document_ids: Optional list of document IDs to filter.
-            tags: Optional list of tags to filter.
-
-        Returns:
-            Deletion OpenSearch query.
-        """
-        filters = [{"term": {"users": user_sub}}]
-        if document_ids:
-            filters.append({"ids": {"values": document_ids}})
-        if tags:
-            filters.append({"terms": {"tags": tags}})
-        return {"bool": {"must": filters}}
-
 
 class SearchDocumentView(ResourceServerMixin, views.APIView):
     """
@@ -309,6 +148,26 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
     authentication_classes = [ResourceServerAuthentication]
     permission_classes = [IsAuthAuthenticated]
 
+    def _get_opensearch_indices(self, audience, services):
+        # Get request user service
+        try:
+            user_service = Service.objects.get(client_id=audience, is_active=True)
+        except Service.DoesNotExist as e:
+            logger.warning("Login failed: No service %s found", audience)
+            raise SuspiciousOperation("Service is not available") from e
+
+        # Find allowed sub-services for this service
+        allowed_services = set(user_service.services.values_list("name", flat=True))
+        allowed_services.add(user_service.name)
+
+        if services:
+            available = set(services).intersection(allowed_services)
+
+            if len(available) < len(services):
+                raise SuspiciousOperation("Some requested services are not available")
+
+        return allowed_services
+
     def post(self, request, *args, **kwargs):
         """
         Handle POST requests to perform a search on indexed documents with optional filtering
@@ -326,20 +185,17 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
             The search query string. This is a required parameter.
         reach : str, optional
             Filter results based on the 'reach' field.
-        tags : List[str], optional
-            Filter results based on the 'tags' field. Documents matching any of the
-            provided tags will be returned.
-        path : str, optional
-            Filter results based on the 'path' field. Only documents whose path
-            starts with the provided value will be returned.
         order_by : str, optional
             Order results by 'relevance', 'created_at', 'updated_at', or 'size'.
             Defaults to 'relevance' if not specified.
         order_direction : str, optional
             Order direction, 'asc' for ascending or 'desc' for descending.
             Defaults to 'desc'.
-        nb_results : int, optional
-            The number of results to return.
+        page_number : int, optional
+            The page number to retrieve.
+            Defaults to 1 if not specified.
+        page_size : int, optional
+            The number of results to return per page.
             Defaults to 50 if not specified.
         services: List[str], optional
             List of services on which we intend to run the query (current service if left empty)
@@ -347,13 +203,6 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
             List of public/authenticated documents the user has visited to limit
             the document returned to the ones the current user has seen.
             Built from linkreach list of a document in docs app.
-        search_type : str, optional
-            Type of search to perform: 'hybrid' or 'full_text'.
-            - 'hybrid': Uses hybrid search if enabled on the server,
-                otherwise falls back to full-text search.
-            - 'full_text': Uses only full-text search, even if hybrid search is enabled
-                on the server.
-            if the not specified, the server will use hybrid search when enabled
 
         Returns:
         --------
@@ -365,38 +214,102 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
         audience = self._get_service_provider_audience()
         user_sub = self.request.user.sub
         groups = []
+        # //////////////////////////////////////////////////
+
+        # Extract and validate query parameters using Pydantic schema
         params = schemas.SearchQueryParametersSchema(**request.data)
 
+        # Compute pagination parameters
+        from_value = (params.page_number - 1) * params.page_size
+        size_value = params.page_size
+
         # Get index list for search query
         try:
-            search_indices = get_opensearch_indices(audience, services=params.services)
+            search_indices = self._get_opensearch_indices(
+                audience, services=params.services
+            )
         except SuspiciousOperation as e:
-            logger.error(e, exc_info=True)
-            return Response(
-                {"detail": "Invalid request."},
-                status=status.HTTP_400_BAD_REQUEST,
+            return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST)
+
+        # Prepare the search query
+        search_body = {
+            "_source": enums.SOURCE_FIELDS,  # limit the fields to return
+            "script_fields": {
+                "number_of_users": {"script": {"source": "doc['users'].size()"}},
+                "number_of_groups": {"script": {"source": "doc['groups'].size()"}},
+            },
+            "query": {"bool": {"must": [], "filter": []}},
+            "sort": [],
+            "from": from_value,
+            "size": size_value,
+        }
+
+        # Adding the text query
+        if params.q == "*":
+            search_body["query"]["bool"]["must"].append({"match_all": {}})
+        else:
+            search_body["query"]["bool"]["must"].append(
+                {
+                    "multi_match": {
+                        "query": params.q,
+                        # Give title more importance over content by a power of 3
+                        "fields": ["title.text^3", "content"],
+                    }
+                }
             )
 
-        logger.info("Search '%s' on indices %s", params.q, search_indices)
-        result = search(
-            q=params.q,
-            nb_results=params.nb_results,
-            order_by=params.order_by,
-            order_direction=params.order_direction,
-            search_indices=search_indices,
-            reach=params.reach,
-            visited=params.visited,
-            user_sub=user_sub,
-            groups=groups,
-            tags=params.tags,
-            path=params.path,
-            search_type=params.search_type
-            if params.search_type
-            else SearchTypeEnum.HYBRID
-            if check_hybrid_search_enabled()
-            else SearchTypeEnum.FULL_TEXT,
-        )["hits"]["hits"]
-        logger.info("found %d results", len(result))
-        logger.debug("results %s", result)
+        # Add sorting logic based on relevance or specified field
+        if params.order_by == enums.RELEVANCE:
+            search_body["sort"].append({"_score": {"order": params.order_direction}})
+        else:
+            search_body["sort"].append(
+                {params.order_by: {"order": params.order_direction}}
+            )
 
-        return Response(result, status=status.HTTP_200_OK)
+        # Apply access control based on documents reach
+        search_body["query"]["bool"]["must"].append(
+            {
+                "bool": {
+                    "should": [
+                        # Access control on public & authenticated reach
+                        {
+                            "bool": {
+                                "must_not": {
+                                    "term": {enums.REACH: enums.ReachEnum.RESTRICTED},
+                                },
+                                # Limit search to already visited documents.
+                                "must": {
+                                    "terms": {
+                                        "_id": sorted(params.visited),
+                                    }
+                                },
+                            },
+                        },
+                        # Access control on restricted search : either user or group should match
+                        {"term": {enums.USERS: user_sub}},
+                        {"terms": {enums.GROUPS: groups}},
+                    ],
+                    # At least one of the 2 optional should clauses must apply
+                    "minimum_should_match": 1,
+                }
+            }
+        )
+
+        # Optional filter by reach if explicitly provided in the query
+        if params.reach is not None:
+            search_body["query"]["bool"]["filter"].append(
+                {"term": {enums.REACH: params.reach}}
+            )
+
+        # Always filter out inactive documents
+        search_body["query"]["bool"]["filter"].append({"term": {"is_active": True}})
+
+        response = client.search(  # pylint: disable=unexpected-keyword-arg
+            index=",".join(search_indices),
+            body=search_body,
+            # Argument added by the query_params() decorator of opensearch and
+            # not in the method declaration.
+            ignore_unavailable=True,
+        )
+
+        return Response(response["hits"]["hits"], status=status.HTTP_200_OK)

src/backend/demo/defaults.py

@@ -1,21 +1,10 @@
 """Parameters that define how the demo site will be built."""
 
-NB_OBJECTS = {"documents": 1000, "services": 5}
+NB_OBJECTS = {
+    "documents": 1000,
+    "services": 5
+}
 
 DEV_SERVICES = (
-    {
-        "name": "docs",
-        "client_id": "impress",
-        "token": "find-api-key-for-docs-with-exactly-50-chars-length",
-    },
-    {
-        "name": "drive",
-        "client_id": "drive",
-        "token": "find-api-key-for-driv-with-exactly-50-chars-length",
-    },
-    {
-        "name": "conversations",
-        "client_id": "conversations",
-        "token": "find-api-key-for-conv-with-exactly-50-chars-length",
-    },
+    {"name": "docs", "client_id": "impress", "token": "find-api-key-for-docs-with-exactly-50-chars-length"},
 )

src/backend/demo/management/commands/create_demo.py

@@ -1,4 +1,4 @@
-# ruff: noqa: S311
+# ruff: noqa: S311, S106
 """create_demo management command"""
 
 import logging
@@ -14,9 +14,7 @@ from django.utils.text import slugify
 from faker import Faker
 from opensearchpy.helpers import bulk
 
-from core import enums, factories
-from core.services.indexing import ensure_index_exists
-from core.services.opensearch import opensearch_client
+from core import enums, factories, opensearch
 
 from demo import defaults
 
@@ -38,7 +36,7 @@ class BulkIndexing:
 
     def bulk_index(self):
         """Actually index documents in bulk to OpenSearch."""
-        _success, failed = bulk(opensearch_client(), self.actions, stats_only=False)
+        _success, failed = bulk(opensearch.client, self.actions, stats_only=False)
 
         if failed:
             self.handle_failures(failed)
@@ -128,8 +126,8 @@ def generate_document():
     )
 
     return {
-        "title.en": fake.sentence(nb_words=10, variable_nb_words=True),
-        "content.en": "\n".join(fake.paragraphs(nb=5)),
+        "title": fake.sentence(nb_words=10, variable_nb_words=True),
+        "content": "\n".join(fake.paragraphs(nb=5)),
         "created_at": created_at,
         "updated_at": updated_at,
         "size": random.randint(0, 100 * 1024**2),
@@ -143,8 +141,7 @@ def create_demo(stdout):
     """
     Create a database with demo data for developers to work in a realistic environment.
     """
-    opensearch_client_ = opensearch_client()
-    opensearch_client_.indices.delete(index="*")
+    opensearch.client.indices.delete("*")
 
     with Timeit(stdout, "Creating services"):
         services = factories.ServiceFactory.create_batch(
@@ -152,8 +149,8 @@ def create_demo(stdout):
         )
 
         for service in services:
-            ensure_index_exists(service.name)
-            opensearch_client_.indices.refresh(index=service.name)
+            opensearch.ensure_index_exists(service.name)
+            opensearch.client.indices.refresh(index=service.name)
 
     with Timeit(stdout, "Creating documents"):
         actions = BulkIndexing(stdout)
@@ -166,14 +163,14 @@ def create_demo(stdout):
     with Timeit(stdout, "Creating dev services"):
         for conf in defaults.DEV_SERVICES:
             service = factories.ServiceFactory(**conf)
-            ensure_index_exists(service.name)
-            opensearch_client_.indices.refresh(index=service.name)
+            opensearch.ensure_index_exists(service.name)
+            opensearch.client.indices.refresh(index=service.name)
 
     # Check and report on indexed documents
     total_indexed = 0
     for service in services:
-        opensearch_client_.indices.refresh(index=service.name)
-        indexed = opensearch_client_.count(index=service.name)["count"]
+        opensearch.client.indices.refresh(index=service.name)
+        indexed = opensearch.client.count(index=service.name)["count"]
         stdout.write(f"  - {service.name:s}: {indexed:d} documents")
         total_indexed += indexed
 

src/backend/demo/tests/test_commands_create_demo.py

@@ -7,8 +7,7 @@ from django.test import override_settings
 
 import pytest
 
-from core import models
-from core.services.opensearch import opensearch_client
+from core import models, opensearch
 
 from demo import defaults
 
@@ -26,11 +25,5 @@ def test_commands_create_demo():
     """The create_demo management command should create objects as expected."""
     call_command("create_demo")
 
-    assert models.Service.objects.exclude(name="docs").count() == 4
-    assert opensearch_client().count()["count"] == 4
-
-    docs = models.Service.objects.get(name="docs")
-    assert docs.client_id == "impress"
-
-    drive = models.Service.objects.get(name="drive")
-    assert drive.client_id == "drive"
+    assert models.Service.objects.count() == 2
+    assert opensearch.client.count()["count"] == 4

src/backend/evaluation/data/corpus/simple_corpus.py

@@ -1,903 +0,0 @@
-"""a simple corpus of documents for evaluation"""
-
-documents = [
-    {
-        "id": "sc-1",
-        "title": "La Révolution Française 1789",
-        "content": (
-            "La Révolution française débute le 14 juillet 1789 avec la prise de la "
-            "Bastille, symbole de l'absolutisme royal. Les causes sont multiples : crise "
-            "financière, famine, influence des Lumières et inégalités sociales criantes. "
-            "L'Assemblée Constituante abolit les privilèges le 4 août et adopte la "
-            "Déclaration des Droits de l'Homme le 26 août. Le roi Louis XVI est exécuté "
-            "en 1793 après la proclamation de la République. Cette révolution transforme "
-            "profondément la France et influence l'Europe entière."
-        ),
-        "updated_at": "2026-02-24T12:00:00Z",
-    },
-    {
-        "id": "sc-2",
-        "title": "La Première Guerre Mondiale",
-        "content": (
-            "La Première Guerre mondiale (1914-1918) oppose les Alliés (France, Royaume- "
-            "Uni, Russie) aux Empires centraux (Allemagne, Autriche-Hongrie). "
-            "L'assassinat de l'archiduc François-Ferdinand à Sarajevo déclenche le "
-            "conflit par le jeu des alliances. La guerre de tranchées caractérise le "
-            "front occidental avec Verdun et la Somme. Les nouvelles armes (gaz, tanks, "
-            "aviation) causent 18 millions de morts. Le traité de Versailles en 1919 "
-            "redessine la carte de l'Europe."
-        ),
-    },
-    {
-        "id": "sc-3",
-        "title": "La Décolonisation en Afrique",
-        "content": (
-            "La décolonisation de l'Afrique s'accélère après 1945, affaiblie par la "
-            "Seconde Guerre mondiale. Le Ghana obtient son indépendance en 1957, suivi de "
-            "17 pays en 1960, 'l'année de l'Afrique'. Le processus varie : négociation "
-            "pacifique (Afrique francophone) ou luttes armées (Algérie, Kenya). Les "
-            "frontières héritées de la colonisation créent des tensions ethniques. Les "
-            "nouveaux États font face à des défis économiques et politiques majeurs."
-        ),
-    },
-    {
-        "id": "sc-4",
-        "title": "Le Réchauffement Climatique",
-        "content": (
-            "Le réchauffement climatique désigne l'augmentation de la température moyenne "
-            "terrestre causée par les émissions de gaz à effet de serre. Depuis l'ère "
-            "industrielle, la température a augmenté de 1,2°C. Les conséquences incluent "
-            "fonte des glaciers, montée des eaux, événements météorologiques extrêmes et "
-            "migrations climatiques. Les accords de Paris en 2015 visent à limiter le "
-            "réchauffement à 1,5°C. La transition énergétique et les énergies "
-            "renouvelables sont essentielles."
-        ),
-    },
-    {
-        "id": "sc-5",
-        "title": "L'Empire Romain",
-        "content": (
-            "L'Empire romain domine le bassin méditerranéen pendant cinq siècles, de 27 "
-            "avant J.-C. à 476 après J.-C. Auguste, premier empereur, établit la Pax "
-            "Romana qui apporte prospérité et stabilité. Rome construit routes, aqueducs "
-            "et monuments grandioses comme le Colisée. Le christianisme devient religion "
-            "d'État en 380. Les invasions barbares et les crises internes provoquent la "
-            "chute de l'Empire d'Occident en 476."
-        ),
-    },
-    {
-        "id": "sc-6",
-        "title": "La Renaissance Européenne",
-        "content": (
-            "La Renaissance (XIVe-XVIe siècles) marque un renouveau artistique, "
-            "scientifique et intellectuel en Europe. Née en Italie avec Florence et les "
-            "Médicis, elle redécouvre l'Antiquité gréco-romaine. Léonard de Vinci, "
-            "Michel-Ange et Raphaël révolutionnent l'art. Gutenberg invente l'imprimerie "
-            "en 1450, facilitant la diffusion des savoirs. Les grandes découvertes "
-            "ouvrent le monde avec Colomb et Magellan. L'humanisme place l'homme au "
-            "centre de la réflexion."
-        ),
-    },
-    {
-        "id": "sc-7",
-        "title": "La Guerre Froide",
-        "content": (
-            "La Guerre Froide (1947-1991) oppose les États-Unis capitalistes à l'URSS "
-            "communiste sans conflit direct. Le rideau de fer divise l'Europe entre blocs "
-            "occidental et oriental. Les crises majeures incluent Berlin (1948, 1961), "
-            "Cuba (1962) et le Vietnam. La course aux armements nucléaires menace "
-            "l'humanité. La détente des années 1970 laisse place aux tensions des années "
-            "1980. La chute du mur de Berlin en 1989 symbolise la fin de la Guerre "
-            "Froide."
-        ),
-    },
-    {
-        "id": "sc-8",
-        "title": "L'Égypte Ancienne",
-        "content": (
-            "L'Égypte ancienne prospère pendant 3000 ans le long du Nil, de 3100 avant "
-            "J.-C. à 30 avant J.-C. Les pharaons sont considérés comme des dieux vivants. "
-            "Les pyramides de Gizeh, notamment celle de Khéops, témoignent de prouesses "
-            "architecturales. L'écriture hiéroglyphique permet l'administration et la "
-            "transmission des savoirs. Le culte des morts et la momification préparent à "
-            "la vie après la mort. Cléopâtre VII est la dernière souveraine avant la "
-            "conquête romaine."
-        ),
-    },
-    {
-        "id": "sc-9",
-        "title": "La Mondialisation",
-        "content": (
-            "La mondialisation désigne l'intégration croissante des économies et sociétés "
-            "à l'échelle planétaire. Accélérée depuis 1990, elle s'appuie sur les "
-            "nouvelles technologies, les transports et la libéralisation des échanges. "
-            "Les firmes multinationales organisent la production mondiale. Les flux "
-            "commerciaux, financiers et migratoires s'intensifient. Ce processus crée des "
-            "interdépendances mais accentue aussi les inégalités entre pays développés et "
-            "en développement."
-        ),
-    },
-    {
-        "id": "sc-10",
-        "title": "La Conquête Spatiale",
-        "content": (
-            "La conquête spatiale débute en 1957 avec le satellite soviétique Spoutnik. "
-            "Youri Gagarine devient le premier homme dans l'espace en 1961. Les "
-            "Américains répondent avec le programme Apollo : Neil Armstrong marche sur la "
-            "Lune en 1969. La Station Spatiale Internationale symbolise la coopération "
-            "internationale depuis 1998. Aujourd'hui, les ambitions incluent Mars, et des "
-            "entreprises privées comme SpaceX révolutionnent l'accès à l'espace."
-        ),
-    },
-    {
-        "id": "sc-11",
-        "title": "Les Métropoles Mondiales",
-        "content": (
-            "Les métropoles mondiales comme New York, Londres, Tokyo et Paris concentrent "
-            "pouvoir économique, politique et culturel. Elles abritent sièges sociaux des "
-            "multinationales, bourses financières et institutions internationales. Ces "
-            "villes sont des nœuds de transport et communication globaux. Elles attirent "
-            "flux migratoires et talents créant diversité mais aussi gentrification. Les "
-            "mégapoles du Sud (Mumbai, São Paulo) connaissent une urbanisation rapide "
-            "avec des défis d'infrastructures."
-        ),
-    },
-    {
-        "id": "sc-12",
-        "title": "La Seconde Guerre Mondiale",
-        "content": (
-            "La Seconde Guerre mondiale (1939-1945) est le conflit le plus meurtrier de "
-            "l'histoire avec 60 millions de morts. Hitler envahit la Pologne en septembre "
-            "1939, déclenchant la guerre. La France capitule en 1940 tandis que le "
-            "Royaume-Uni résiste. L'Allemagne attaque l'URSS en 1941 et le Japon Pearl "
-            "Harbor. Le débarquement en Normandie en 1944 libère l'Europe. La Shoah "
-            "extermine 6 millions de Juifs. Le conflit se termine avec les bombes "
-            "atomiques sur Hiroshima et Nagasaki."
-        ),
-    },
-    {
-        "id": "sc-13",
-        "title": "Le Développement Durable",
-        "content": (
-            "Le développement durable vise à concilier croissance économique, protection "
-            "environnementale et équité sociale. Défini au Sommet de Rio en 1992, il "
-            "répond aux besoins présents sans compromettre ceux des générations futures. "
-            "Les 17 Objectifs de Développement Durable de l'ONU incluent éradication de "
-            "la pauvreté, éducation, égalité des genres et action climatique. Les "
-            "énergies renouvelables, l'économie circulaire et la consommation responsable "
-            "sont des leviers essentiels."
-        ),
-    },
-    {
-        "id": "sc-14",
-        "title": "La Chine contemporaine",
-        "content": (
-            "La Chine est devenue la deuxième économie mondiale grâce aux réformes de "
-            "Deng Xiaoping en 1978. L'ouverture au capitalisme maintient le régime "
-            "communiste du Parti unique. Le pays est 'l'usine du monde' avec des zones "
-            "économiques spéciales. Les nouvelles routes de la soie étendent son "
-            "influence internationale. Avec 1,4 milliard d'habitants, la Chine fait face "
-            "à des défis environnementaux, démographiques et sociaux majeurs."
-        ),
-    },
-    {
-        "id": "sc-15",
-        "title": "Les Grandes Découvertes",
-        "content": (
-            "Les grandes découvertes (XVe-XVIe siècles) voient les Européens explorer le "
-            "monde. Christophe Colomb atteint l'Amérique en 1492, croyant trouver les "
-            "Indes. Vasco de Gama ouvre la route des Indes en 1498. Magellan réalise le "
-            "premier tour du monde en 1522. Ces expéditions sont motivées par la "
-            "recherche d'épices, d'or et l'évangélisation. Elles inaugurent la "
-            "colonisation européenne et l'échange colombien qui transforment les "
-            "civilisations."
-        ),
-    },
-    {
-        "id": "sc-16",
-        "title": "L'Union Européenne",
-        "content": (
-            "L'Union européenne naît du désir de paix après 1945. Le traité de Rome en "
-            "1957 crée la CEE avec six membres fondateurs. L'UE compte aujourd'hui 27 "
-            "États après le Brexit. Le marché unique permet libre circulation des biens, "
-            "services, capitaux et personnes. L'euro est adopté par 20 pays. L'UE fait "
-            "face à des défis : crise migratoire, euroscepticisme et divergences "
-            "économiques entre Nord et Sud."
-        ),
-    },
-    {
-        "id": "sc-17",
-        "title": "La Révolution Industrielle",
-        "content": (
-            "La révolution industrielle débute en Angleterre au XVIIIe siècle avec la "
-            "machine à vapeur de Watt. L'industrie textile mécanisée précède la "
-            "métallurgie et les chemins de fer. Le charbon et la vapeur fournissent "
-            "l'énergie nécessaire. L'urbanisation s'accélère avec l'exode rural vers les "
-            "usines. Les conditions ouvrières misérables suscitent les mouvements sociaux "
-            "et le syndicalisme. Cette révolution transforme radicalement l'économie et "
-            "la société européennes."
-        ),
-    },
-    {
-        "id": "sc-18",
-        "title": "Les Inégalités Nord-Sud",
-        "content": (
-            "Les inégalités Nord-Sud opposent pays développés et pays en développement. "
-            "L'Indice de Développement Humain mesure richesse, éducation et santé. Les "
-            "pays du Nord concentrent richesses et technologies tandis que le Sud subit "
-            "pauvreté et dépendance économique. L'héritage colonial, l'endettement et les "
-            "termes de l'échange inégaux perpétuent ces écarts. Les pays émergents "
-            "(BRICS) remettent en question cette division binaire."
-        ),
-    },
-    {
-        "id": "sc-19",
-        "title": "Le Siècle des Lumières",
-        "content": (
-            "Le Siècle des Lumières (XVIIIe siècle) est un mouvement intellectuel "
-            "européen promouvant raison, science et progrès. Voltaire critique "
-            "l'intolérance religieuse, Montesquieu théorise la séparation des pouvoirs, "
-            "Rousseau défend la souveraineté populaire. L'Encyclopédie de Diderot et "
-            "d'Alembert diffuse les savoirs. Ces philosophes remettent en cause "
-            "l'absolutisme et les privilèges. Leurs idées inspirent les révolutions "
-            "américaine et française."
-        ),
-    },
-    {
-        "id": "sc-20",
-        "title": "Les Ressources Énergétiques",
-        "content": (
-            "Les ressources énergétiques sont inégalement réparties sur la planète. Les "
-            "énergies fossiles (pétrole, gaz, charbon) dominent mais sont non "
-            "renouvelables et polluantes. Le Moyen-Orient concentre 48% des réserves "
-            "pétrolières mondiales. La Russie utilise le gaz comme arme géopolitique. Les "
-            "énergies renouvelables (solaire, éolien, hydraulique) progressent face au "
-            "réchauffement climatique. La transition énergétique est un défi majeur du "
-            "XXIe siècle pour assurer souveraineté et durabilité."
-        ),
-    },
-    {
-        "id": "sc-21",
-        "title": "Le Lion d'Afrique",
-        "content": (
-            "Le lion est le plus grand félin d'Afrique et vit en groupe social appelé "
-            "troupe. Les mâles se distinguent par leur majestueuse crinière qui protège "
-            "leur cou lors des combats. Les lions chassent principalement au crépuscule, "
-            "les lionnes étant les principales chasseuses. Ils peuvent rugir jusqu'à 8 "
-            "kilomètres de distance pour marquer leur territoire. Malheureusement, leur "
-            "population a diminué de 43% en 20 ans."
-        ),
-    },
-    {
-        "id": "sc-22",
-        "title": "Le Dauphin",
-        "content": (
-            "Les dauphins sont parmi les animaux les plus intelligents de la planète. Ils "
-            "communiquent par des clics et sifflements complexes, chaque individu "
-            "possédant sa propre signature sonore. Vivant en groupes sociaux "
-            "sophistiqués, ils chassent en coordination et s'entraident. Les dauphins "
-            "peuvent nager jusqu'à 30 km/h et plonger à 300 mètres de profondeur. Leur "
-            "cerveau possède plus de circonvolutions que celui de l'homme."
-        ),
-    },
-    {
-        "id": "sc-23",
-        "title": "L'Éléphant d'Asie",
-        "content": (
-            "L'éléphant d'Asie est légèrement plus petit que son cousin africain mais "
-            "tout aussi majestueux. Ces herbivores peuvent consommer jusqu'à 150 kg de "
-            "végétation par jour et boire 140 litres d'eau. Leur trompe contient 40 000 "
-            "muscles et leur sert à manger, boire, se laver et communiquer. Les éléphants "
-            "vivent en matriarcat dirigé par la femelle la plus âgée. Leur mémoire "
-            "exceptionnelle leur permet de se souvenir des points d'eau sur de vastes "
-            "territoires."
-        ),
-    },
-    {
-        "id": "sc-24",
-        "title": "Le Colibri",
-        "content": (
-            "Le colibri est le plus petit oiseau du monde, certaines espèces ne pesant "
-            "que 2 grammes. Ses ailes battent jusqu'à 80 fois par seconde, lui permettant "
-            "de faire du vol stationnaire et même de voler en arrière. Son métabolisme "
-            "est si rapide qu'il doit manger jusqu'à deux fois son poids en nectar "
-            "quotidiennement. Le colibri peut visiter 1000 fleurs par jour. Ses couleurs "
-            "iridescentes changent selon l'angle de la lumière."
-        ),
-    },
-    {
-        "id": "sc-25",
-        "title": "L'Ours polaire",
-        "content": (
-            "L'ours polaire est le plus grand carnivore terrestre, pouvant peser jusqu'à "
-            "800 kg. Sa fourrure blanche et sa peau noire lui permettent d'absorber la "
-            "chaleur du soleil. Son odorat est si développé qu'il peut détecter un phoque "
-            "sous 1 mètre de glace à 1,5 km. Le réchauffement climatique menace gravement "
-            "son habitat."
-        ),
-    },
-    {
-        "id": "sc-26",
-        "title": "Le Caméléon",
-        "content": (
-            "Le caméléon possède la capacité extraordinaire de changer de couleur selon "
-            "son humeur, la température et la communication sociale. Ses yeux peuvent "
-            "bouger indépendamment l'un de l'autre à 360 degrés. Sa langue peut s'étendre "
-            "jusqu'à deux fois la longueur de son corps pour capturer des insectes. "
-            "Certaines espèces mesurent seulement 3 cm tandis que d'autres atteignent 70 "
-            "cm. Ils se déplacent lentement et de manière saccadée pour imiter le "
-            "mouvement des feuilles."
-        ),
-    },
-    {
-        "id": "sc-27",
-        "title": "Le Manchot Empereur",
-        "content": (
-            "Le manchot empereur survit aux conditions les plus extrêmes de la planète en "
-            "Antarctique. Les mâles couvent l'œuf unique pendant 64 jours dans des "
-            "températures atteignant -40°C sans manger. Ils se regroupent en tortue pour "
-            "se protéger du froid, tournant régulièrement pour partager la chaleur. Ces "
-            "oiseaux peuvent plonger jusqu'à 500 mètres de profondeur et retenir leur "
-            "respiration 22 minutes. Leur parade nuptiale comprend des chants complexes "
-            "et synchronisés."
-        ),
-    },
-    {
-        "id": "sc-28",
-        "title": "Le Requin Blanc",
-        "content": (
-            "Le grand requin blanc est l'un des prédateurs marins les plus redoutés et "
-            "fascinants. Il peut atteindre 6 mètres de long et peser plus de 2 tonnes. "
-            "Ses dents triangulaires et dentelées se renouvellent constamment tout au "
-            "long de sa vie. Il peut détecter une goutte de sang dans 100 litres d'eau "
-            "grâce à son odorat exceptionnel. Contrairement à sa réputation, il attaque "
-            "rarement l'homme, préférant les phoques et les otaries."
-        ),
-    },
-    {
-        "id": "sc-29",
-        "title": "L'Abeille",
-        "content": (
-            "Les abeilles jouent un rôle crucial dans la pollinisation de 80% des plantes "
-            "à fleurs. Une ruche peut contenir jusqu'à 60 000 abeilles organisées en "
-            "société matriarcale autour de la reine. Les ouvrières communiquent la "
-            "localisation des fleurs par une danse complexe en forme de huit. Une abeille "
-            "produit environ 1/12 de cuillère à café de miel dans sa vie. Leur déclin "
-            "inquiétant menace notre sécurité alimentaire."
-        ),
-    },
-    {
-        "id": "sc-30",
-        "title": "Le Guépard",
-        "content": (
-            "Le guépard est l'animal terrestre le plus rapide, capable d'atteindre 110 "
-            "km/h en quelques secondes. Contrairement aux autres félins, il ne peut pas "
-            "rétracter complètement ses griffes, ce qui lui donne une meilleure "
-            "adhérence. Sa queue longue lui sert de balancier lors des changements de "
-            "direction. Son corps élancé et ses poumons surdimensionnés sont optimisés "
-            "pour la course. Il chasse le jour pour éviter les autres prédateurs plus "
-            "puissants."
-        ),
-    },
-    {
-        "id": "sc-31",
-        "title": "La Pieuvre",
-        "content": (
-            "La pieuvre possède trois cœurs et un sang bleu à base de cuivre. Son "
-            "intelligence remarquable lui permet de résoudre des problèmes complexes et "
-            "d'utiliser des outils. Chacun de ses huit bras contient des neurones, lui "
-            "donnant une forme de pensée décentralisée. Elle peut changer de couleur et "
-            "de texture instantanément pour se camoufler. Certaines espèces peuvent "
-            "passer à travers des ouvertures de la taille d'une pièce de monnaie."
-        ),
-    },
-    {
-        "id": "sc-32",
-        "title": "Le Koala",
-        "content": (
-            "Le koala dort jusqu'à 20 heures par jour pour économiser l'énergie "
-            "nécessaire à digérer l'eucalyptus toxique. Il se nourrit exclusivement de "
-            "feuilles d'eucalyptus, pouvant distinguer entre 600 espèces différentes. Son "
-            "système digestif unique contient des bactéries spéciales pour neutraliser "
-            "les toxines. Les bébés koalas naissent de la taille d'un haricot et restent "
-            "6 mois dans la poche maternelle. Ils ne boivent presque jamais, tirant l'eau "
-            "des feuilles qu'ils consomment."
-        ),
-    },
-    {
-        "id": "sc-33",
-        "title": "Le Gorille des Montagnes",
-        "content": (
-            "Le gorille des montagnes est l'un de nos plus proches parents, partageant "
-            "98% de notre ADN. Les mâles silverback peuvent peser jusqu'à 200 kg et sont "
-            "d'une force exceptionnelle. Malgré leur puissance, ce sont des animaux "
-            "pacifiques et végétariens. Ils vivent en groupes familiaux dirigés par un "
-            "mâle dominant et communiquent par 25 vocalisations différentes. Moins de "
-            "1000 individus survivent dans les forêts d'Afrique centrale."
-        ),
-    },
-    {
-        "id": "sc-34",
-        "title": "Le Papillon Monarque",
-        "content": (
-            "Le papillon monarque effectue une migration annuelle de 4000 km entre le "
-            "Canada et le Mexique. Aucun individu ne complète le voyage entier, il faut 4 "
-            "générations pour accomplir le cycle. Ils utilisent le soleil comme boussole "
-            "et possèdent une horloge circadienne sophistiquée. Leur couleur orange vif "
-            "avertit les prédateurs de leur toxicité acquise en mangeant de l'asclépiade. "
-            "Des millions de papillons se regroupent dans quelques forêts mexicaines en "
-            "hiver."
-        ),
-    },
-    {
-        "id": "sc-35",
-        "title": "Le Loup Gris",
-        "content": (
-            "Le loup gris vit en meute organisée hiérarchiquement autour d'un couple "
-            "alpha. Ces chasseurs coopératifs peuvent traquer des proies bien plus "
-            "grandes qu'eux grâce à leur coordination. Ils communiquent par hurlements "
-            "pouvant porter jusqu'à 10 km dans les forêts. Un loup peut parcourir 70 km "
-            "en une nuit à la recherche de nourriture. Leur réintroduction dans certains "
-            "écosystèmes a démontré leur rôle crucial dans l'équilibre naturel."
-        ),
-    },
-    {
-        "id": "sc-36",
-        "title": "Le Kangourou Roux",
-        "content": (
-            "Le kangourou roux est le plus grand marsupial au monde, pouvant atteindre 2 "
-            "mètres de haut. Il peut sauter jusqu'à 9 mètres de long et maintenir une "
-            "vitesse de 50 km/h. Sa queue musclée lui sert de trépied au repos et de "
-            "balancier en mouvement. Les femelles peuvent retarder le développement d'un "
-            "embryon si les conditions sont défavorables. Parfaitement adapté au climat "
-            "aride australien, il peut passer des mois sans boire."
-        ),
-    },
-    {
-        "id": "sc-37",
-        "title": "La Baleine à Bosse",
-        "content": (
-            "Les baleines à bosse produisent des chants complexes pouvant durer 20 "
-            "minutes et s'entendre à des centaines de kilomètres. Les mâles d'une même "
-            "région partagent le même chant qui évolue chaque année. Malgré leur masse de "
-            "30 tonnes, elles peuvent sauter entièrement hors de l'eau. Elles migrent sur "
-            "25 000 km par an entre zones d'alimentation et de reproduction. Leurs "
-            "nageoires pectorales sont les plus longues de tous les cétacés."
-        ),
-    },
-    {
-        "id": "sc-38",
-        "title": "Le Serpent Python",
-        "content": (
-            "Le python réticulé peut atteindre 9 mètres de long, ce qui en fait l'un des "
-            "plus longs serpents. Il tue sa proie par constriction, resserrant son "
-            "étreinte à chaque expiration de la victime. Ces serpents peuvent passer des "
-            "mois sans manger après avoir avalé une grande proie. Leurs organes "
-            "sensoriels thermiques leur permettent de détecter la chaleur corporelle dans "
-            "l'obscurité. Contrairement aux idées reçues, ils ne peuvent pas avaler un "
-            "humain adulte."
-        ),
-    },
-    {
-        "id": "sc-39",
-        "title": "Le Hibou Grand-Duc",
-        "content": (
-            "Le hibou grand-duc est le plus grand rapace nocturne d'Europe avec une "
-            "envergure de 2 mètres. Son vol silencieux est rendu possible par la "
-            "structure spéciale de ses plumes. Il peut tourner sa tête à 270 degrés grâce "
-            "à 14 vertèbres cervicales. Sa vision nocturne est 100 fois plus sensible que "
-            "celle de l'humain. Son hululement puissant peut s'entendre jusqu'à 2 km. Il "
-            "chasse des proies variées, du mulot au renardeau."
-        ),
-    },
-    {
-        "id": "sc-40",
-        "title": "Le Paresseux",
-        "content": (
-            "Le paresseux se déplace si lentement que des algues poussent sur sa "
-            "fourrure, lui donnant un camouflage verdâtre. Il descend de son arbre une "
-            "fois par semaine pour déféquer, moment où il est le plus vulnérable. Son "
-            "métabolisme est le plus lent de tous les non-hibernants. Il peut tourner sa "
-            "tête à 270 degrés pour surveiller les prédateurs. Les paresseux passent 90% "
-            "de leur vie suspendus la tête en bas et peuvent même dormir ainsi."
-        ),
-    },
-    {
-        "id": "sc-41",
-        "title": "Bœuf Bourguignon",
-        "content": (
-            "Le bœuf bourguignon est un plat mijoté emblématique de la cuisine française. "
-            "Coupez 1,5 kg de bœuf en cubes et faites-les revenir dans du beurre. Ajoutez "
-            "2 carottes, 2 oignons, 3 gousses d'ail et un bouquet garni. Versez 75 cl de "
-            "vin rouge de Bourgogne et 25 cl de bouillon. Laissez mijoter 3 heures à feu "
-            "doux. Ajoutez 200g de champignons et 150g de lardons dorés. Servez avec des "
-            "pommes de terre vapeur ou des tagliatelles fraîches."
-        ),
-    },
-    {
-        "id": "sc-42",
-        "title": "Ratatouille Provençale",
-        "content": (
-            "Tranchez finement 2 aubergines, 2 courgettes, 2 poivrons, 4 tomates et 1 "
-            "oignon. Dans une cocotte, faites revenir l'oignon et l'ail dans de l'huile "
-            "d'olive. Ajoutez les légumes par couches en alternant. Assaisonnez avec du "
-            "thym, du romarin, sel et poivre. Couvrez et laissez mijoter 40 minutes à feu "
-            "doux. La ratatouille peut se déguster chaude ou froide, accompagnée de riz "
-            "ou de pain grillé."
-        ),
-    },
-    {
-        "id": "sc-43",
-        "title": "Coq au Vin",
-        "content": (
-            "Découpez un coq ou un poulet fermier en morceaux. Faites mariner 12 heures "
-            "dans 75 cl de vin rouge avec carottes, oignons et aromates. Égouttez et "
-            "faites dorer les morceaux avec 100g de lardons. Flambez au cognac puis "
-            "ajoutez la marinade filtrée. Laissez mijoter 1h30. Ajoutez des champignons "
-            "et des petits oignons grelots. Liez la sauce avec du beurre manié. Servez "
-            "avec des croûtons aillés."
-        ),
-    },
-    {
-        "id": "sc-44",
-        "title": "Tarte Tatin",
-        "content": (
-            "Dans un moule à tarte, faites fondre 100g de beurre avec 100g de sucre pour "
-            "créer un caramel. Disposez 6 pommes Reinette coupées en quartiers en rosace "
-            "serrée. Enfournez 30 minutes à 180°C. Recouvrez de pâte feuilletée et "
-            "enfournez 25 minutes supplémentaires. Laissez tiédir 10 minutes avant de "
-            "retourner sur un plat. Servez tiède avec de la crème fraîche ou une boule de "
-            "glace vanille."
-        ),
-    },
-    {
-        "id": "sc-45",
-        "title": "Bouillabaisse Marseillaise",
-        "content": (
-            "Faites suer dans l'huile d'olive 2 oignons, 4 tomates, 4 gousses d'ail et du "
-            "fenouil. Ajoutez safran, thym, laurier et zeste d'orange. Versez 2 litres de "
-            "fumet de poisson et portez à ébullition. Ajoutez 2 kg de poissons variés "
-            "(rascasse, grondin, rouget) coupés en morceaux. Cuisez 15 minutes à gros "
-            "bouillons. Servez avec des croûtons, de la rouille et du gruyère râpé."
-        ),
-    },
-    {
-        "id": "sc-46",
-        "title": "Quiche Lorraine",
-        "content": (
-            "Garnissez un moule de pâte brisée. Faites revenir 200g de lardons fumés sans "
-            "matière grasse. Battez 4 œufs avec 30 cl de crème fraîche, sel, poivre et "
-            "noix de muscade. Disposez les lardons sur la pâte et versez l'appareil. "
-            "Ajoutez éventuellement 100g de gruyère râpé. Enfournez 35 minutes à 180°C "
-            "jusqu'à ce que la garniture soit dorée et gonflée. Servez tiède avec une "
-            "salade verte."
-        ),
-    },
-    {
-        "id": "sc-47",
-        "title": "Blanquette de Veau",
-        "content": (
-            "Coupez 1,2 kg d'épaule de veau en cubes. Blanchissez-les 5 minutes à l'eau "
-            "bouillante puis rincez. Remettez dans une cocotte avec carottes, oignons "
-            "piqués de clous de girofle et bouquet garni. Couvrez d'eau et laissez "
-            "mijoter 1h30. Préparez un roux avec beurre et farine, ajoutez le bouillon "
-            "filtré. Liez avec 2 jaunes d'œufs et 20 cl de crème. Ajoutez champignons et "
-            "oignons grelots. Servez avec du riz."
-        ),
-    },
-    {
-        "id": "sc-48",
-        "title": "Crêpes Bretonnes",
-        "content": (
-            "Mélangez 250g de farine, 4 œufs, 50 cl de lait et une pincée de sel. Ajoutez "
-            "50g de beurre fondu et laissez reposer 1 heure. Huilez légèrement une poêle "
-            "chaude et versez une louche de pâte. Faites cuire 2 minutes de chaque côté "
-            "jusqu'à ce que la crêpe soit dorée. Garnissez de sucre, confiture, chocolat "
-            "ou jambon-fromage selon vos envies. Servez immédiatement."
-        ),
-    },
-    {
-        "id": "sc-49",
-        "title": "Cassoulet Toulousain",
-        "content": (
-            "Faites tremper 500g de haricots blancs 12 heures. Cuisez-les 1 heure avec "
-            "carottes, oignons et bouquet garni. Faites revenir 4 cuisses de canard "
-            "confites, 400g de saucisse de Toulouse et 200g de poitrine fumée. Mélangez "
-            "haricots et viandes dans une cocotte avec la graisse de canard. Couvrez de "
-            "chapelure et enfournez 1 heure à 160°C. Cassez la croûte plusieurs fois "
-            "pendant la cuisson."
-        ),
-    },
-    {
-        "id": "sc-50",
-        "title": "Soufflé au Fromage",
-        "content": (
-            "Préparez une béchamel avec 40g de beurre, 40g de farine et 25 cl de lait. "
-            "Hors du feu, incorporez 150g de gruyère râpé et 4 jaunes d'œufs. Montez 5 "
-            "blancs en neige ferme avec une pincée de sel. Incorporez délicatement les "
-            "blancs à la préparation. Versez dans des ramequins beurrés et farinés. "
-            "Enfournez 20 minutes à 180°C sans ouvrir le four. Servez immédiatement."
-        ),
-    },
-    {
-        "id": "sc-51",
-        "title": "Pot-au-feu",
-        "content": (
-            "Dans une grande marmite, placez 1,5 kg de viande de bœuf (gîte, paleron, "
-            "plat de côtes). Couvrez d'eau froide et portez à ébullition en écumant. "
-            "Ajoutez 4 carottes, 4 poireaux, 2 navets, 2 oignons piqués de clous de "
-            "girofle et un bouquet garni. Laissez mijoter 3 heures à feu doux. Servez le "
-            "bouillon en entrée puis la viande et les légumes avec cornichons, moutarde "
-            "et gros sel."
-        ),
-    },
-    {
-        "id": "sc-52",
-        "title": "Mousse au Chocolat",
-        "content": (
-            "Faites fondre 200g de chocolat noir au bain-marie. Séparez 6 œufs et "
-            "incorporez les jaunes au chocolat fondu tiède. Montez les blancs en neige "
-            "ferme avec une pincée de sel et 20g de sucre. Incorporez délicatement les "
-            "blancs au chocolat en trois fois avec une spatule. Répartissez dans des "
-            "verrines et réfrigérez 4 heures minimum. Décorez de copeaux de chocolat ou "
-            "de chantilly."
-        ),
-    },
-    {
-        "id": "sc-53",
-        "title": "Gratin Dauphinois",
-        "content": (
-            "Épluchez et émincez finement 1,5 kg de pommes de terre. Frottez un plat à "
-            "gratin avec une gousse d'ail. Disposez les pommes de terre en couches en "
-            "assaisonnant de sel, poivre et noix de muscade. Mélangez 50 cl de crème avec "
-            "25 cl de lait et versez sur les pommes de terre. Ajoutez quelques noisettes "
-            "de beurre. Enfournez 1h15 à 160°C jusqu'à obtenir une croûte dorée."
-        ),
-    },
-    {
-        "id": "sc-54",
-        "title": "Salade de légumes",
-        "content": (
-            "Disposez sur un plat 4 tomates en quartiers, 1 concombre émincé, 1 poivron "
-            "coupé, 200g de haricots verts cuits, 4 œufs durs, 100g d'olives noires et 8 "
-            "filets d'anchois. Préparez une vinaigrette avec huile d'olive, vinaigre de "
-            "vin, moutarde, ail et basilic. Arrosez généreusement et servez frais. "
-            "Certains ajoutent des artichauts ou des fèves."
-        ),
-    },
-    {
-        "id": "sc-55",
-        "title": "Tarte au Citron Meringuée",
-        "content": (
-            "Garnissez un moule de pâte sablée et faites-la cuire à blanc 15 minutes. "
-            "Préparez une crème avec 4 jaunes d'œufs, 150g de sucre, le zeste et jus de 3 "
-            "citrons. Cuisez à feu doux en remuant jusqu'à épaississement. Versez sur la "
-            "pâte. Montez 4 blancs en neige avec 100g de sucre. Couvrez la tarte de "
-            "meringue en formant des pointes. Enfournez 10 minutes à 150°C pour dorer."
-        ),
-    },
-    {
-        "id": "sc-56",
-        "title": "Magret de Canard aux Figues",
-        "content": (
-            "Quadrillez la peau de 2 magrets sans entamer la chair. Salez et poivrez. "
-            "Posez-les côté peau dans une poêle froide. Faites cuire 7 minutes puis "
-            "retournez pour 4 minutes (rosé). Réservez au chaud. Dans la graisse, faites "
-            "revenir 8 figues coupées en deux avec 2 cuillères de miel et un trait de "
-            "vinaigre balsamique. Tranchez les magrets et nappez de sauce aux figues."
-        ),
-    },
-    {
-        "id": "sc-57",
-        "title": "Clafoutis aux Cerises",
-        "content": (
-            "Disposez 500g de cerises lavées et équeutées (traditionnellement non "
-            "dénoyautées) dans un plat beurré. Battez 4 œufs avec 100g de sucre jusqu'à "
-            "ce que le mélange blanchisse. Ajoutez 100g de farine, 50 cl de lait et une "
-            "pincée de sel. Versez l'appareil sur les cerises. Enfournez 45 minutes à "
-            "180°C. Saupoudrez de sucre glace et servez tiède ou froid."
-        ),
-    },
-    {
-        "id": "sc-58",
-        "title": "Fondue Savoyarde",
-        "content": (
-            "Frottez un caquelon avec une gousse d'ail coupée. Versez 15 cl de vin blanc "
-            "sec et faites chauffer. Ajoutez 400g de comté, 400g de beaufort et 200g de "
-            "reblochon râpés. Remuez en forme de 8 jusqu'à ce que le fromage fonde. "
-            "Ajoutez 5 cl de kirsch et du poivre. Maintenez au chaud sur un réchaud. "
-            "Trempez des cubes de pain rassis piqués sur des fourchettes."
-        ),
-    },
-    {
-        "id": "sc-59",
-        "title": "Crème Brûlée",
-        "content": (
-            "Faites chauffer 50 cl de crème avec une gousse de vanille fendue. Battez 6 "
-            "jaunes d'œufs avec 100g de sucre. Versez la crème chaude en remuant puis "
-            "filtrez. Répartissez dans des ramequins. Cuisez au bain-marie 40 minutes à "
-            "150°C. Réfrigérez 4 heures. Saupoudrez généreusement de sucre et caramélisez "
-            "au chalumeau ou sous le gril. Laissez durcir 2 minutes avant de servir."
-        ),
-    },
-    {
-        "id": "sc-60",
-        "title": "Tartare de Bœuf",
-        "content": (
-            "Hachez finement 600g de filet de bœuf bien frais au couteau. Ajoutez 2 "
-            "échalotes ciselées, 2 cuillères de câpres, 4 cornichons hachés, persil et "
-            "ciboulette. Assaisonnez avec moutarde, sauce Worcestershire, Tabasco, sel et "
-            "poivre. Incorporez 2 jaunes d'œufs et un filet d'huile d'olive. Formez des "
-            "dômes et servez avec des frites maison et une salade verte. Se prépare au "
-            "dernier moment."
-        ),
-    },
-    {
-        "id": "sc-61",
-        "title": "La Musique",
-        "content": (
-            "La musique est l'art d'organiser les sons dans le temps selon le rythme, la "
-            "mélodie et l'harmonie. Présente dans toutes les cultures, elle accompagne "
-            "les rites, les émotions et les récits humains depuis la préhistoire. Des "
-            "modes antiques grecs au contrepoint baroque, du jazz à l'électro, elle "
-            "évolue sans cesse. Les compositeurs explorent les timbres et les structures "
-            "tandis que l’improvisation garde la spontanéité vivante. La musique relie "
-            "mathématique, émotion et mouvement."
-        ),
-    },
-    {
-        "id": "sc-62",
-        "title": "Le Cinéma",
-        "content": (
-            "Le cinéma est l'art de raconter des histoires par le mouvement des images et "
-            "le son. Né à la fin du XIXe siècle avec les frères Lumière, il a rapidement "
-            "fusionné technique et poésie. Le montage, la lumière et la mise en scène en "
-            "font un art total mêlant littérature, théâtre et musique. Du muet de Chaplin "
-            "au cinéma numérique, chaque époque invente un nouveau langage visuel. Le "
-            "cinéma explore la mémoire, les rêves et la condition humaine à travers "
-            "l’écran."
-        ),
-    },
-    {
-        "id": "sc-63",
-        "title": "La Danse",
-        "content": (
-            "La danse est l’art du mouvement du corps dans l’espace et le temps, souvent "
-            "accompagné de musique. Elle exprime des émotions, raconte des histoires ou "
-            "célèbre des rites. Des danses tribales aux ballets classiques, des danses "
-            "contemporaines au hip-hop, chaque culture invente ses gestes et son rythme. "
-            "La chorégraphie unit discipline et liberté, le corps devenant un instrument "
-            "expressif. La danse relie énergie, esthétique et communication non verbale."
-        ),
-    },
-    {
-        "id": "sc-64",
-        "title": "La Peinture à l'Huile",
-        "content": (
-            "La peinture à l'huile est une technique artistique utilisant des pigments "
-            "mélangés à de l'huile siccative, généralement de lin. Inventée au XVe siècle "
-            "et perfectionnée par les maîtres flamands comme Van Eyck, elle permet des "
-            "glacis subtils et des dégradés lumineux. Le temps de séchage lent offre la "
-            "possibilité de travailler les transitions et les détails. Les grands maîtres "
-            "comme Rembrandt, Vermeer et plus tard les impressionnistes ont exploité ses "
-            "possibilités. Cette technique reste aujourd'hui la plus prisée pour la "
-            "peinture de chevalet."
-        ),
-    },
-    {
-        "id": "sc-65",
-        "title": "La Sculpture sur Pierre",
-        "content": (
-            "La sculpture sur pierre est l'un des arts les plus anciens de l'humanité, "
-            "remontant à la préhistoire. Le sculpteur taille directement dans le marbre, "
-            "le granit ou le calcaire avec des ciseaux et des masses. Michel-Ange "
-            "considérait que la statue existait déjà dans le bloc, il suffisait de "
-            "libérer la forme. Cette technique soustractive ne pardonne pas l'erreur. Les "
-            "œuvres comme le David ou la Pietà démontrent la capacité de donner vie et "
-            "émotion à la pierre froide."
-        ),
-    },
-    {
-        "id": "sc-66",
-        "title": "La Calligraphie",
-        "content": (
-            "La calligraphie est l'art de former les lettres avec beauté et harmonie. En "
-            "Occident, les moines copistes médiévaux ont perfectionné l'onciale et la "
-            "gothique. En Asie, la calligraphie chinoise et japonaise est considérée "
-            "comme la forme d'art la plus pure, où chaque trait exprime l'énergie et "
-            "l'esprit de l'artiste. L'outil traditionnel est le pinceau ou le calame. La "
-            "maîtrise nécessite des années de pratique pour contrôler la pression, la "
-            "vitesse et le rythme."
-        ),
-    },
-    {
-        "id": "sc-67",
-        "title": "La Photographie",
-        "content": (
-            "La photographie transforme la capture d'images en expression créative depuis "
-            "le XIXe siècle. Des pionniers comme Ansel Adams et Henri Cartier-Bresson ont "
-            "élevé le médium au rang d'art majeur. La composition, la lumière, le cadrage "
-            "et le moment décisif sont essentiels. Le passage au numérique a ouvert de "
-            "nouvelles possibilités de post-traitement. La photographie d'art explore "
-            "tous les genres : portrait, paysage, abstrait, documentaire et conceptuel."
-        ),
-    },
-    {
-        "id": "sc-68",
-        "title": "La Danse Contemporaine",
-        "content": (
-            "La danse contemporaine émerge au XXe siècle comme rupture avec le ballet "
-            "classique. Des chorégraphes comme Martha Graham, Merce Cunningham et Pina "
-            "Bausch explorent de nouveaux langages corporels. Cette forme privilégie "
-            "l'expression émotionnelle, la liberté de mouvement et l'improvisation. Le "
-            "corps devient un outil de questionnement social et politique. Les spectacles "
-            "intègrent souvent d'autres disciplines comme la vidéo, le théâtre et la "
-            "musique expérimentale."
-        ),
-    },
-    {
-        "id": "sc-69",
-        "title": "L'Origami",
-        "content": (
-            "L'origami est l'art japonais du pliage de papier, transformant une feuille "
-            "plane en sculpture tridimensionnelle sans couper ni coller. Pratiqué depuis "
-            "le VIe siècle au Japon, il était d'abord réservé aux cérémonies religieuses. "
-            "Les modèles traditionnels incluent la grue (symbole de paix), la grenouille "
-            "et la fleur. L'origami moderne explore la complexité mathématique avec des "
-            "créations hyperréalistes. Cette discipline développe patience, précision et "
-            "compréhension spatiale."
-        ),
-    },
-    {
-        "id": "sc-70",
-        "title": "La Mosaïque",
-        "content": (
-            "La mosaïque assemble de petits fragments colorés (tesselles) de pierre, "
-            "céramique ou verre pour créer des images et motifs. Les Romains et Byzantins "
-            "ont porté cet art à son apogée avec les splendeurs de Ravenne et de "
-            "Constantinople. Chaque tesselle est posée individuellement sur un support "
-            "avec du mortier. Les jeux de lumière sur les tesselles de verre créent des "
-            "effets lumineux uniques. Gaudi a réinventé la mosaïque moderne avec le "
-            "trencadis au Parc Güell."
-        ),
-    },
-    {
-        "id": "sc-71",
-        "title": "Le Théâtre",
-        "content": (
-            "Le théâtre est à la fois un art de la représentation et un lieu de rencontre "
-            "sociale. Né dans l'Antiquité grecque avec les tragédies d'Eschyle et "
-            "Sophocle, il explore les grandes questions humaines. Le Moyen Âge voit "
-            "l'essor des mystères religieux, tandis que la Renaissance célèbre "
-            "Shakespeare et Molière. Le théâtre moderne expérimente avec le réalisme, "
-            "l'absurde et le théâtre de l'opprimé. Il combine texte, jeu d'acteur, décor "
-            "et lumière pour créer une expérience immersive."
-        ),
-    },
-    {
-        "id": "sc-72",
-        "title": "Le Vitrail",
-        "content": (
-            "Le vitrail assemble des morceaux de verre coloré maintenus par des baguettes "
-            "de plomb pour créer des compositions lumineuses. Au Moyen Âge, les "
-            "cathédrales gothiques comme Chartres transforment la lumière divine en "
-            "récits bibliques. Les maîtres verriers maîtrisent la chimie des oxydes "
-            "métalliques pour obtenir des couleurs intenses. Chaque pièce est taillée "
-            "selon le carton préparatoire puis sertie. Le XXe siècle voit Chagall et "
-            "Soulages réinventer cet art millénaire."
-        ),
-    },
-    {
-        "id": "sc-73",
-        "title": "La Gravure",
-        "content": (
-            "La gravure est une technique d'impression où l'artiste incise une matrice "
-            "(bois, métal, pierre) pour créer des estampes multiples. La xylogravie "
-            "(bois) est la plus ancienne, utilisée par Dürer et les estampes japonaises "
-            "ukiyo-e. La taille-douce (métal) comprend l'eau-forte, l'aquatinte et le "
-            "burin, prisées par Rembrandt et Goya. La lithographie, inventée en 1796, "
-            "permet des nuances subtiles exploitées par Toulouse-Lautrec. Chaque tirage "
-            "est numéroté et signé par l'artiste."
-        ),
-    },
-]

src/backend/evaluation/data/full_text/__init__.py

@@ -1,4 +0,0 @@
-"""
-evaluation dataset for full_text capabilities.
-evaluation should be good with embeddings disabled
-"""

src/backend/evaluation/data/full_text/documents.py

@@ -1,10 +0,0 @@
-"""document data for full text evaluation"""
-
-from ..corpus.simple_corpus import documents as simple_corpus_documents
-
-documents = [
-    *simple_corpus_documents,
-    {"id": "ft-1", "title": "L'éléphant", "content": "L'éléphant s'est échappé"},
-    {"id": "ft-2", "title": "Foot", "content": "Le foot est un sport populaire"},
-    {"id": "ft-3", "title": "Il va courir", "content": "Il va courir"},
-]

src/backend/evaluation/data/full_text/queries.py

@@ -1,26 +0,0 @@
-"""Queries and expected document IDs for evaluation in French language."""
-
-queries = [
-    {
-        "q": "elephant",
-        "expected_document_ids": ["sc-23", "ft-1"],
-    },
-    {
-        "q": "courir",
-        "expected_document_ids": ["ft-3"],
-    },
-    {
-        # test "football"  -> "foot"
-        "q": "football",
-        "expected_document_ids": ["ft-2"],
-    },
-    {  # test partial word matching
-        "q": "couri",
-        "expected_document_ids": ["ft-3"],
-    },
-    {
-        # test fuzzy matching with ngrams
-        "q": "courrir",
-        "expected_document_ids": ["ft-3"],
-    },
-]

src/backend/evaluation/data/semantic/__init__.py

@@ -1 +0,0 @@
-"""base evaluation datasets for semantic capabilities."""

src/backend/evaluation/data/semantic/documents.py

@@ -1,5 +0,0 @@
-"""Documents for semantic evaluation."""
-
-from ..corpus.simple_corpus import documents as simple_corpus_documents
-
-documents = simple_corpus_documents

src/backend/evaluation/data/semantic/queries.py

@@ -1,32 +0,0 @@
-"""Queries and expected document IDs for evaluation in French language."""
-
-queries = [
-    {
-        "q": "cours d'histoire de l'antiquité",
-        "expected_document_ids": ["sc-5", "sc-8"],
-    },
-    {
-        "q": "recette salée végétarienne",
-        "expected_document_ids": ["sc-42", "sc-54", "sc-58"],
-    },
-    {
-        "q": "art dramatique",
-        "expected_document_ids": ["sc-71"],
-    },
-    {
-        "q": "art de bouger son corps",
-        "expected_document_ids": ["sc-63", "sc-68"],
-    },
-    {
-        "q": "mammifères aquatiques",
-        "expected_document_ids": ["sc-22", "sc-37"],
-    },
-    {
-        "q": "insectes pollinisateurs",
-        "expected_document_ids": ["sc-29"],
-    },
-    {
-        "q": "prédateur félin",
-        "expected_document_ids": ["sc-21", "sc-30"],
-    },
-]

src/backend/evaluation/data/v0/__init__.py

@@ -1 +0,0 @@
-"""evaluation dataset tests"""

src/backend/evaluation/data/v0/documents.py

@@ -1,3 +0,0 @@
-"""document data"""
-
-documents = [{"id": 1, "title": "document", "content": "a document"}]

src/backend/evaluation/data/v0/queries.py

@@ -1,8 +0,0 @@
-"""Queries and expected document IDs for test evaluation."""
-
-queries = [
-    {
-        "q": "a query",
-        "expected_document_ids": [1],
-    },
-]

src/backend/evaluation/management/commands/evaluate_search_engine.py

@@ -1,231 +0,0 @@
-"""
-Evaluate search engine performance with test documents and queries.
-"""
-
-import importlib
-import logging
-import math
-
-from django.core.management.base import BaseCommand
-
-from core.enums import SearchTypeEnum
-from core.management.commands.create_search_pipeline import (
-    ensure_search_pipeline_exists,
-)
-from core.services.opensearch import (
-    check_hybrid_search_enabled,
-    opensearch_client,
-)
-from core.services.search import (
-    search,
-)
-from core.utils import (
-    bulk_create_documents,
-    delete_index,
-    delete_search_pipeline,
-    get_language_value,
-    prepare_index,
-)
-
-logger = logging.getLogger(__name__)
-
-
-class Command(BaseCommand):
-    """Evaluate search engine performance"""
-
-    help = __doc__
-    opensearch_client_ = opensearch_client()
-    index_name = "evaluation-index"
-    search_params = {
-        "nb_results": 20,
-        "order_by": "relevance",
-        "order_direction": "desc",
-        "search_indices": {index_name},
-        "reach": None,
-        "user_sub": "user_sub",
-        "groups": [],
-        "visited": [],
-        "tags": [],
-    }
-    base_data_path = "evaluation/data"
-    documents = []
-    queries = []
-    id_to_title = {}
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            dest="dataset_name",
-            type=str,
-            help="Name of the dataset to use for evaluation",
-        )
-        parser.add_argument(
-            "--min_score",
-            dest="min_score",
-            type=float,
-            default=0.0,
-            help="hits with a score lower than min_score are ignored",
-        )
-        parser.add_argument(
-            "--keep-index",
-            dest="keep_index",
-            type=bool,
-            default=True,
-            help="If True the index is not dropped after evaluation.",
-        )
-        parser.add_argument(
-            "--force-reindex",
-            dest="force_reindex",
-            type=bool,
-            default=False,
-            help=(
-                "If True the index is dropped and recreated from scratch before evaluation."
-            ),
-        )
-
-    def handle(self, *args, **options):
-        """Launch the search engine evaluation."""
-
-        self.init_evaluation(options["dataset_name"], options["force_reindex"])
-        self.stdout.write(
-            f"[INFO] Starting evaluation with {len(self.documents)} "
-            f"documents and {len(self.queries)} queries"
-        )
-
-        evaluations = [
-            self.evaluate_query(query, options["min_score"]) for query in self.queries
-        ]
-
-        avg_metrics = self.calculate_average_metrics(evaluations)
-        self.stdout.write(
-            f"\n{'=' * 60}\n"
-            f"[SUMMARY] Average Performance\n"
-            f"{'=' * 60}\n"
-            f"  Average NDCG: {avg_metrics['avg_ndcg']:.2%}\n"
-            f"  Average Precision: {avg_metrics['avg_precision']:.2%}\n"
-            f"  Average Recall: {avg_metrics['avg_recall']:.2%}\n"
-            f"  Average F1-Score: {avg_metrics['avg_f1_score']:.2%}\n"
-        )
-
-        self.close_evaluation(options["keep_index"])
-        self.stdout.write(self.style.SUCCESS("\n[SUCCESS] Evaluation completed"))
-
-    def init_evaluation(self, dataset_name, force_reindex):
-        """Initialize evaluation by preparing index and mapping."""
-        self.documents = (
-            importlib.import_module(f"evaluation.data.{dataset_name}.documents")
-        ).documents
-        self.queries = (
-            importlib.import_module(f"evaluation.data.{dataset_name}.queries")
-        ).queries
-        self.id_to_title = {
-            document["id"]: document["title"] for document in self.documents
-        }
-        check_hybrid_search_enabled.cache_clear()
-        delete_search_pipeline()
-        ensure_search_pipeline_exists()
-        if not opensearch_client().indices.exists(index=self.index_name):
-            prepare_index(self.index_name, bulk_create_documents(self.documents))
-        elif force_reindex:
-            delete_index(self.index_name)
-            prepare_index(self.index_name, bulk_create_documents(self.documents))
-
-    def evaluate_query(self, query, min_score=0.0):
-        """Evaluate a single query and return metrics."""
-        results = search(
-            q=query["q"],
-            search_type=SearchTypeEnum.HYBRID
-            if check_hybrid_search_enabled()
-            else SearchTypeEnum.FULL_TEXT,
-            **self.search_params,
-        )
-        expected_titles = [
-            self.id_to_title[document_id]
-            for document_id in query["expected_document_ids"]
-        ]
-        retrieved_ordered_titles = [
-            get_language_value(result["_source"], "title")
-            for result in results["hits"]["hits"]
-            if result["_score"] >= min_score
-        ]
-
-        metrics = self.calculate_metrics(expected_titles, retrieved_ordered_titles)
-
-        self.stdout.write(
-            f"  [QUERY EVALUATION]\n"
-            f"  q: {query['q']}\n"
-            f"  expect: {list(expected_titles)}\n"
-            f"  result: {list(retrieved_ordered_titles)}\n"
-            f"  NDCG: {metrics['ndcg']:.2%} \n"
-            f"  PRECISION: {metrics['precision']:.2%} \n"
-            f"  RECALL: {metrics['recall']:.2%} \n"
-            f"  F1-SCORE: {metrics['f1_score']:.2%} \n"
-        )
-        return {
-            "q": query["q"],
-            "expected_titles": expected_titles,
-            "retrieved_titles": retrieved_ordered_titles,
-            "metrics": metrics,
-        }
-
-    def calculate_metrics(self, expected_titles, retrieved_ordered_titles):
-        """Calculate precision, recall, F1-score, DCG and NDCG."""
-
-        dcg = self.calculate_dcg(expected_titles, retrieved_ordered_titles)
-        idcg = self.calculate_dcg(expected_titles, expected_titles)
-        ndcg = dcg / idcg if idcg > 0 else 0
-        nb_true_positives = len(set(expected_titles) & set(retrieved_ordered_titles))
-        precision = (
-            nb_true_positives / len(retrieved_ordered_titles)
-            if retrieved_ordered_titles
-            else 0
-        )
-        recall = nb_true_positives / len(expected_titles) if expected_titles else 0
-        f1_score = (
-            2 * (precision * recall) / (precision + recall)
-            if (precision + recall) > 0
-            else 0
-        )
-
-        return {
-            "ndcg": ndcg,
-            "precision": precision,
-            "recall": recall,
-            "f1_score": f1_score,
-            "true_positives": nb_true_positives,
-        }
-
-    def calculate_dcg(self, expected_titles, retrieved_ordered_titles):
-        """Calculate Discounted Cumulative Gain."""
-        return sum(
-            (1 if title in expected_titles else 0) / math.log2(rank + 2)
-            for rank, title in enumerate(retrieved_ordered_titles)
-        ) / len(expected_titles)
-
-    def calculate_average_metrics(self, evaluations):
-        """Calculate average metrics across all queries."""
-        if not evaluations:
-            return {
-                "avg_ndcg": 0,
-                "avg_precision": 0,
-                "avg_recall": 0,
-                "avg_f1_score": 0,
-            }
-
-        total_ndcg = sum(r["metrics"]["ndcg"] for r in evaluations)
-        total_precision = sum(r["metrics"]["precision"] for r in evaluations)
-        total_recall = sum(r["metrics"]["recall"] for r in evaluations)
-        total_f1 = sum(r["metrics"]["f1_score"] for r in evaluations)
-        nb_evaluations = len(evaluations)
-
-        return {
-            "avg_ndcg": total_ndcg / nb_evaluations,
-            "avg_precision": total_precision / nb_evaluations,
-            "avg_recall": total_recall / nb_evaluations,
-            "avg_f1_score": total_f1 / nb_evaluations,
-        }
-
-    def close_evaluation(self, keep_index):
-        """Delete the evaluation index."""
-        delete_search_pipeline()
-        if not keep_index:
-            delete_index(self.index_name)

src/backend/evaluation/tests/test_evaluate_search_engine.py

@@ -1,134 +0,0 @@
-"""
-Test suite for evaluate_search_engine management command
-"""
-
-import io
-import logging
-from unittest.mock import patch
-
-from django.core.management import call_command
-
-import pytest
-import responses
-
-from core.services.opensearch import check_hybrid_search_enabled, opensearch_client
-from core.utils import delete_index, delete_search_pipeline
-
-logger = logging.getLogger(__name__)
-
-pytestmark = pytest.mark.django_db
-
-
-INDEX_NAME = "evaluation-index"
-
-
-@pytest.fixture(autouse=True)
-def clear_caches_and_cleanup():
-    """Clear caches and cleanup before and after each test"""
-    clear()
-    yield
-    clear()
-
-
-@pytest.fixture(autouse=True)
-def disable_hybrid_search(settings):
-    """Disable hybrid search for all tests to prevent API calls"""
-    settings.HYBRID_SEARCH_ENABLED = False
-
-
-def clear():
-    """Clear caches and delete index and pipeline"""
-    check_hybrid_search_enabled.cache_clear()
-    delete_search_pipeline()
-    delete_index(INDEX_NAME)
-
-
-def assert_output_successful(output):
-    """Assert that the output indicates a successful evaluation"""
-    assert "[INFO] Starting evaluation with 1 documents and 1 queries" in output
-    assert "[QUERY EVALUATION]" in output
-    assert "q: a query" in output
-    assert "[SUMMARY] Average Performance" in output
-    assert "Average NDCG:" in output
-    assert "Average Precision:" in output
-    assert "Average Recall:" in output
-    assert "Average F1-Score:" in output
-    assert "[SUCCESS] Evaluation completed" in output
-
-
-@responses.activate
-def test_evaluate_search_engine_command_v0():
-    """Test running the evaluate_search_engine command with v0 dataset"""
-    out = io.StringIO()
-    call_command(
-        "evaluate_search_engine",
-        "v0",
-        stdout=out,
-    )
-
-    assert_output_successful(out.getvalue())
-
-    # Index should still exist because keep-index is True by default
-    assert opensearch_client().indices.exists(index="evaluation-index")
-
-
-@responses.activate
-def test_evaluate_search_engine_command_without_keep_index():
-    """Test that keep-index option False erases index"""
-    out = io.StringIO()
-    call_command(
-        "evaluate_search_engine",
-        "v0",
-        keep_index=False,
-        stdout=out,
-    )
-
-    assert_output_successful(out.getvalue())
-
-    # Index should not exist
-    assert not opensearch_client().indices.exists(index="evaluation-index")
-
-
-@patch("evaluation.management.commands.evaluate_search_engine.delete_index")
-@responses.activate
-def test_evaluate_search_engine_command_force_reindex(mock_delete_index):
-    """Test that force-reindex must delete and recreates the index"""
-    out = io.StringIO()
-    # run once to create the index
-    call_command(
-        "evaluate_search_engine",
-        "v0",
-        stdout=out,
-    )
-
-    mock_delete_index.clear()
-    # Run again with force-reindex
-    call_command(
-        "evaluate_search_engine",
-        "v0",
-        force_reindex=True,
-        stdout=out,
-    )
-
-    # Verify delete_index was called once with the correct index name
-    mock_delete_index.assert_called_once_with("evaluation-index")
-
-
-@responses.activate
-def test_evaluate_search_engine_min_score_filter():
-    """Test that min_score filters out low-scoring results"""
-
-    out = io.StringIO()
-    super_high_score = 1000.0
-    call_command(
-        "evaluate_search_engine",
-        "v0",
-        min_score=super_high_score,
-        stdout=out,
-    )
-
-    # Assert all scores are null proving all results were filtered out
-    assert (
-        "NDCG: 0.00% \n  PRECISION: 0.00% \n  RECALL: 0.00% \n  F1-SCORE: 0.00%"
-        in out.getvalue()
-    )

src/backend/find/settings.py

@@ -19,7 +19,6 @@ from django.utils.translation import gettext_lazy as _
 import sentry_sdk
 from configurations import Configuration, values
 from sentry_sdk.integrations.django import DjangoIntegration
-from sentry_sdk.integrations.logging import ignore_logger
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -124,24 +123,16 @@ class Base(Configuration):
     # https://docs.djangoproject.com/en/3.1/topics/i18n/
 
     # Languages
+    LANGUAGE_CODE = values.Value("en-us")
+
+    # Careful! Languages should be ordered by priority, as this tuple is used to get
+    # fallback/default languages throughout the app.
     LANGUAGES = values.SingleNestedTupleValue(
         (
-            ("fr", _("French")),
-            ("en", _("English")),
-            ("de", _("German")),
-            ("nl", _("Dutch")),
-            ("und", None),
+            ("en-us", _("English")),
+            ("fr-fr", _("French")),
         )
     )
-    SUPPORTED_LANGUAGE_CODES = tuple(
-        language_code for language_code, _ in LANGUAGES.value
-    )
-    LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD = values.FloatValue(
-        default=0.75,
-        environ_name="LANGUAGE_DETECTION_CONFIDENCE_THRESHOLD",
-        environ_prefix=None,
-    )
-    UNDETERMINED_LANGUAGE_CODE = "und"
 
     LOCALE_PATHS = (os.path.join(BASE_DIR, "locale"),)
 
@@ -196,7 +187,6 @@ class Base(Configuration):
         # find
         "core",
         "demo",
-        "evaluation",
         # Third party apps
         "corsheaders",
         "dockerflow.django",
@@ -253,9 +243,6 @@ class Base(Configuration):
     OPENSEARCH_USE_SSL = values.BooleanValue(
         default=True, environ_name="OPENSEARCH_USE_SSL", environ_prefix=None
     )
-    OPENSEARCH_INDEX_PREFIX = values.Value(
-        default="find", environ_name="OPENSEARCH_INDEX_PREFIX", environ_prefix=None
-    )
 
     SPECTACULAR_SETTINGS = {
         "TITLE": "Find API",
@@ -275,50 +262,6 @@ class Base(Configuration):
 
     AUTH_USER_MODEL = "core.User"
 
-    # Trigrams search settings
-    TRIGRAMS_BOOST = values.Value(
-        default=0.25, environ_name="TRIGRAMS_BOOST", environ_prefix=None
-    )
-    TRIGRAMS_MINIMUM_SHOULD_MATCH = values.Value(
-        default="75%", environ_name="TRIGRAMS_MINIMUM_SHOULD_MATCH", environ_prefix=None
-    )
-
-    # Hybrid Search settings
-    HYBRID_SEARCH_ENABLED = values.BooleanValue(
-        default=False, environ_name="HYBRID_SEARCH_ENABLED", environ_prefix=None
-    )
-    HYBRID_SEARCH_PIPELINE_ID = "hybrid-search-pipeline"
-    HYBRID_SEARCH_WEIGHTS = values.ListValue(
-        default=[0.3, 0.7], environ_name="HYBRID_SEARCH_WEIGHTS", environ_prefix=None
-    )
-    # Multi-embedding: chunk documents and embed each chunk separately
-    CHUNK_SIZE = values.IntegerValue(
-        default=512, environ_name="CHUNK_SIZE", environ_prefix=None
-    )
-    CHUNK_OVERLAP = values.IntegerValue(
-        default=50, environ_name="CHUNK_OVERLAP", environ_prefix=None
-    )
-    EMBEDDING_API_PATH = values.Value(
-        # embedding is the vector representation of a document used for semantic search
-        default="None",
-        environment_name="EMBEDDING_API_PATH",
-        environ_prefix=None,
-    )
-    EMBEDDING_API_KEY = values.Value(
-        default=None, environ_name="EMBEDDING_API_KEY", environ_prefix=None
-    )
-    EMBEDDING_REQUEST_TIMEOUT = values.IntegerValue(
-        default=10, environ_name="EMBEDDING_REQUEST_TIMEOUT", environ_prefix=None
-    )
-    EMBEDDING_API_MODEL_NAME = values.Value(
-        default="embeddings-small",
-        environ_name="EMBEDDING_API_MODEL_NAME",
-        environ_prefix=None,
-    )
-    EMBEDDING_DIMENSION = values.IntegerValue(
-        default=1024, environ_name="EMBEDDING_DIMENSION", environ_prefix=None
-    )
-
     # CORS
     CORS_ALLOW_CREDENTIALS = True
     CORS_ALLOW_ALL_ORIGINS = values.BooleanValue(True)
@@ -326,7 +269,7 @@ class Base(Configuration):
     CORS_ALLOWED_ORIGIN_REGEXES = values.ListValue([])
 
     # Sentry
-    SENTRY_DSN = values.Value(None, environ_name="SENTRY_DSN", environ_prefix=None)
+    SENTRY_DSN = values.Value(None, environ_name="SENTRY_DSN")
 
     # Celery
     CELERY_BROKER_URL = values.Value("redis://redis:6379/0")
@@ -532,11 +475,8 @@ class Base(Configuration):
                 release=get_release(),
                 integrations=[DjangoIntegration()],
             )
-            scope = sentry_sdk.get_current_scope()
-            scope.set_extra("application", "backend")
-
-            # Ignore the logs added by the DockerflowMiddleware
-            ignore_logger("request.summary")
+            with sentry_sdk.configure_scope() as scope:
+                scope.set_extra("application", "backend")
 
 
 class Build(Base):
@@ -614,9 +554,6 @@ class Production(Base):
     """
 
     # Security
-    # Add allowed host from environment variables.
-    # The machine hostname is added by default,
-    # it makes the application pingable by a load balancer on the same machine by example
     ALLOWED_HOSTS = [
         *values.ListValue([], environ_name="ALLOWED_HOSTS"),
         gethostbyname(gethostname()),
@@ -636,14 +573,6 @@ class Production(Base):
     # In other cases, you should comment the following line to avoid security issues.
     # SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
     SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
-    SECURE_HSTS_SECONDS = 60
-    SECURE_HSTS_PRELOAD = True
-    SECURE_HSTS_INCLUDE_SUBDOMAINS = True
-    SECURE_SSL_REDIRECT = True
-    SECURE_REDIRECT_EXEMPT = [
-        "^__lbheartbeat__",
-        "^__heartbeat__",
-    ]
 
     # Modern browsers require to have the `secure` attribute on cookies with `Samesite=none`
     CSRF_COOKIE_SECURE = True
@@ -660,11 +589,6 @@ class Production(Base):
                 environ_name="REDIS_URL",
                 environ_prefix=None,
             ),
-            "TIMEOUT": values.IntegerValue(
-                30,  # timeout in seconds
-                environ_name="CACHES_DEFAULT_TIMEOUT",
-                environ_prefix=None,
-            ),
             "OPTIONS": {
                 "CLIENT_CLASS": "django_redis.client.DefaultClient",
             },

src/backend/pyproject.toml

@@ -17,37 +17,35 @@ classifiers = [
     "License :: OSI Approved :: MIT License",
     "Natural Language :: English",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.10",
 ]
 description = "An application to print markdown to pdf from a set of managed templates."
 keywords = ["Django", "Contacts", "Templates", "RBAC"]
 license = { file = "LICENSE" }
 readme = "README.md"
-requires-python = "~=3.14.3"
+requires-python = ">=3.10"
 dependencies = [
-    "celery[redis]==5.6.2",
+    "celery[redis]==5.5.3",
     "django-configurations==2.5.1",
-    "django-cors-headers==4.9.0",
-    "redis==7.3.0",
+    "django-cors-headers==4.7.0",
+    "redis==5.2.1",
     "django-redis==6.0.0",
-    "django==6.0.3",
-    "django-lasuite[all]==0.0.25",
-    "djangorestframework==3.16.1",
-    "drf_spectacular==0.29.0",
-    "dockerflow==2026.3.4",
-    "factory_boy==3.3.3",
-    "gunicorn==25.1.0",
-    "py3langid==0.3.0",
-    "langchain-text-splitters==1.1.1",
-    "mozilla-django-oidc==5.0.2",
-    "psycopg[binary]==3.3.3",
-    "pydantic==2.12.5",
+    "django==5.2.6",
+    "django-lasuite[all]==0.0.14",
+    "djangorestframework==3.16.0",
+    "drf_spectacular==0.28.0",
+    "dockerflow==2024.4.2",
+    "factory_boy==3.3.1",
+    "gunicorn==23.0.0",
+    "mozilla-django-oidc==4.0.1",
+    "psycopg[binary]==3.2.9",
+    "pydantic==2.10.5",
     "pyjwt==2.10.1",
-    "requests==2.32.5",
-    "sentry-sdk==2.54.0",
-    "url-normalize==2.2.1",
-    "opensearch-py==3.1.0",
-    "whitenoise==6.12.0",
+    "requests==2.32.4",
+    "sentry-sdk==2.32.0",
+    "url-normalize==1.4.3",
+    "opensearch-py==2.8.0",
+    "whitenoise==6.8.2",
 ]
 
 [project.urls]
@@ -59,21 +57,21 @@ dependencies = [
 [project.optional-dependencies]
 dev = [
     "django-extensions==4.1",
-    "drf-spectacular-sidecar==2026.3.1",
-    "faker==40.11.0",
+    "drf-spectacular-sidecar==2025.7.1",
+    "faker==33.3.0",
     "ipdb==0.13.13",
-    "ipython==9.11.0",
-    "pyfakefs==6.1.5",
-    "pylint-django==2.7.0",
-    "pylint==4.0.5",
-    "pytest-cov==7.0.0",
-    "pytest-django==4.12.0",
-    "pytest==9.0.2",
+    "ipython==8.31.0",
+    "pyfakefs==5.9.1",
+    "pylint-django==2.6.1",
+    "pylint==3.3.7",
+    "pytest-cov==6.2.1",
+    "pytest-django==4.11.1",
+    "pytest==8.4.1",
     "pytest-icdiff==0.9",
     "pytest-xdist==3.8.0",
-    "responses==0.26.0",
-    "ruff==0.15.6",
-    "types-requests==2.32.4.20260107",
+    "responses==0.25.7",
+    "ruff==0.12.2",
+    "types-requests==2.32.4.20250611",
 ]
 
 [tool.setuptools]
@@ -91,7 +89,6 @@ exclude = [
     "venv",
     "__pycache__",
     "*/migrations/*",
-    ".vscode*"
 ]
 line-length = 88
 

src/backend/setup.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+"""Setup file for the find module. All configuration stands in the setup.cfg file."""
+# coding: utf-8
+
+from setuptools import setup  # pylint: disable=import-error
+
+setup()

src/helm/find/Chart.yaml

@@ -1,4 +1,4 @@
 apiVersion: v2
 type: application
 name: find
-version: 0.0.3
+version: 0.0.2

src/helm/find/README.md

@@ -43,7 +43,6 @@
 | `backend.shareProcessNamespace`                       | Enable share process namespace between containers                                  | `false`                                         |
 | `backend.sidecars`                                    | Add sidecars containers to backend deployment                                      | `[]`                                            |
 | `backend.migrateJobAnnotations`                       | Annotations for the migrate job                                                    | `{}`                                            |
-| `backend.podSecurityContext`                          | Configure backend Pod security context                                             | `nil`                                           |
 | `backend.securityContext`                             | Configure backend Pod security context                                             | `nil`                                           |
 | `backend.envVars`                                     | Configure backend container environment variables                                  | `undefined`                                     |
 | `backend.envVars.BY_VALUE`                            | Example environment variable by setting value directly                             |                                                 |
@@ -59,15 +58,15 @@
 | `backend.migrate.command`                             | backend migrate command                                                            | `["python","manage.py","migrate","--no-input"]` |
 | `backend.migrate.restartPolicy`                       | backend migrate job restart policy                                                 | `Never`                                         |
 | `backend.probes.liveness.path`                        | Configure path for backend HTTP liveness probe                                     | `/__heartbeat__`                                |
-| `backend.probes.liveness.targetPort`                  | Configure port for backend HTTP liveness probe                                     | `nil`                                           |
+| `backend.probes.liveness.targetPort`                  | Configure port for backend HTTP liveness probe                                     | `undefined`                                     |
 | `backend.probes.liveness.initialDelaySeconds`         | Configure initial delay for backend liveness probe                                 | `10`                                            |
 | `backend.probes.liveness.initialDelaySeconds`         | Configure timeout for backend liveness probe                                       | `10`                                            |
-| `backend.probes.startup.path`                         | Configure path for backend HTTP startup probe                                      | `nil`                                           |
-| `backend.probes.startup.targetPort`                   | Configure port for backend HTTP startup probe                                      | `nil`                                           |
-| `backend.probes.startup.initialDelaySeconds`          | Configure initial delay for backend startup probe                                  | `nil`                                           |
-| `backend.probes.startup.initialDelaySeconds`          | Configure timeout for backend startup probe                                        | `nil`                                           |
+| `backend.probes.startup.path`                         | Configure path for backend HTTP startup probe                                      | `undefined`                                     |
+| `backend.probes.startup.targetPort`                   | Configure port for backend HTTP startup probe                                      | `undefined`                                     |
+| `backend.probes.startup.initialDelaySeconds`          | Configure initial delay for backend startup probe                                  | `undefined`                                     |
+| `backend.probes.startup.initialDelaySeconds`          | Configure timeout for backend startup probe                                        | `undefined`                                     |
 | `backend.probes.readiness.path`                       | Configure path for backend HTTP readiness probe                                    | `/__lbheartbeat__`                              |
-| `backend.probes.readiness.targetPort`                 | Configure port for backend HTTP readiness probe                                    | `nil`                                           |
+| `backend.probes.readiness.targetPort`                 | Configure port for backend HTTP readiness probe                                    | `undefined`                                     |
 | `backend.probes.readiness.initialDelaySeconds`        | Configure initial delay for backend readiness probe                                | `10`                                            |
 | `backend.probes.readiness.initialDelaySeconds`        | Configure timeout for backend readiness probe                                      | `10`                                            |
 | `backend.resources`                                   | Resource requirements for the backend container                                    | `{}`                                            |

src/helm/find/generate-readme.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 docker image ls | grep readme-generator-for-helm
 if [ "$?" -ne "0" ]; then

src/helm/find/templates/backend_deployment.yaml

@@ -90,10 +90,6 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.backend.podSecurityContext }}
-      securityContext:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
       {{- with .Values.backend.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

src/helm/find/templates/backend_job.yaml

@@ -75,10 +75,6 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.backend.podSecurityContext }}
-      securityContext:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
       {{- with .Values.backend.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

src/helm/find/templates/backend_job_createsuperuser.yaml

@@ -76,10 +76,6 @@ spec:
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.backend.podSecurityContext }}
-      securityContext:
-        {{- toYaml . | nindent 8 }}
-      {{- end }}
       {{- with .Values.backend.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

src/helm/find/values.yaml

@@ -92,9 +92,6 @@ backend:
   ## @param backend.migrateJobAnnotations Annotations for the migrate job
   migrateJobAnnotations: {}
 
-  ## @param backend.podSecurityContext Configure backend Pod security context
-  podSecurityContext: null
-
   ## @param backend.securityContext Configure backend Pod security context
   securityContext: null