✅(backend) test

i am testing the rescore feature
✨(backend) add enable_rescore params
2026-03-16 09:51:58 +01:00 · 2026-03-16 09:51:58 +01:00 · 2026-03-16 09:51:58 +01:00 · 2026-03-16 09:51:58 +01:00 · 2026-03-16 09:51:58 +01:00 · 2026-03-10 15:05:14 +01:00
15 changed files with 143 additions and 655 deletions
@@ -9,7 +9,7 @@ and this project adheres to
 # Unreleased

 ## Added
-
+- ✨(backend) add rescore on `updated_at`
 - 👷(docker) add arm64 platform support for image builds
 - ✨(backend) add semantic search
 - ✨(backend) add multi-embedding and chunking
@@ -31,7 +31,6 @@ and this project adheres to
 - ✨(backend) add deletion endpoint
 - ✨(backend) add path filter
 - ✨(backend) add search_type param
- ✨(backend) implement reranker #47

 ## Changed

@@ -43,3 +42,6 @@ and this project adheres to

 - 🐛(backend) fix missing index creation in 'index/' view
 - 🐛(backend) fix parallel test execution issues
+
+## Removed
+- 🗑️(backend) remove sorting
@@ -38,9 +38,6 @@ UPDATED_AT = "updated_at"
 USERS = "users"
 GROUPS = "groups"

-RELEVANCE = "relevance"
-
-ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, REACH)
 SOURCE_FIELDS = (
    TITLE,
    CONTENT,
@@ -1,6 +1,6 @@
 """Pydantic model to validate documents before indexation."""

-from typing import Annotated, List, Literal, Optional
+from typing import Annotated, List, Optional

 from django.utils import timezone
 from django.utils.text import slugify
@@ -116,11 +116,9 @@ class SearchQueryParametersSchema(BaseModel):
    reach: Optional[enums.ReachEnum] = None
    tags: StringListParameter = Field(default_factory=list)
    path: Optional[str] = None
-    order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default=enums.RELEVANCE)
-    order_direction: Optional[Literal["asc", "desc"]] = Field(default="desc")
    nb_results: Optional[conint(ge=1, le=300)] = Field(default=50)
    search_type: Optional[SearchTypeEnum] = Field(default=None)
-    rerank: Optional[bool] = Field(default=None)
+    rescore: bool = Field(default=True)


 class DeleteDocumentsSchema(BaseModel):
@@ -21,7 +21,6 @@ def embed_text(text):
        json={
            "input": text,
            "model": settings.EMBEDDING_API_MODEL_NAME,
-            "dimensions": settings.EMBEDDING_DIMENSION,
            "encoding_format": "float",
        },
        timeout=settings.EMBEDDING_REQUEST_TIMEOUT,
@@ -39,4 +38,14 @@ def embed_text(text):
        logger.warning("unexpected embedding response format: %s", response.text)
        return None

+    if len(embedding) != settings.EMBEDDING_DIMENSION:
+        logger.warning(
+            "unexpected embedding dimension: "
+            "EMBEDDING_DIMENSION is set to %d "
+            "but the configured embedding model returned a vector of dimension %d",
+            settings.EMBEDDING_DIMENSION,
+            len(embedding),
+        )
+        return None
+
    return embedding
@@ -1,128 +0,0 @@
-"""Reranking utilities using rerankers library."""
-
-import logging
-from functools import cache
-
-from django.conf import settings
-
-from rerankers import Reranker  # pylint: disable=import-error
-from rerankers.models.ranker import BaseRanker  # pylint: disable=import-error
-
-from core.services.indexing import format_document
-from core.utils import get_language_value
-
-logger = logging.getLogger(__name__)
-
-
-def rerank(query: str, hits: list[dict]) -> list[dict]:
-    """
-    Rerank search results using the configured reranker model.
-
-    Args:
-        query: The search query string
-        hits: List of OpenSearch hit objects with _source containing title and content
-
-    Returns:
-        List of reranked results in the same format as input with a _reranked_score
-    """
-
-    reranker = get_reranker()
-    if reranker is None:
-        logger.warning("Could not import reranker, returning original results")
-        return hits
-
-    try:
-        return _rerank(reranker, query, hits)
-
-    except Exception as e:  # noqa: BLE001# pylint: disable=broad-exception-caught
-        logger.error("Reranking failed: %s, returning original results", str(e))
-        return hits
-
-
-@cache
-def get_reranker() -> BaseRanker | None:
-    """
-    Get the reranker instance.
-    Returns None if the reranker library is not available or if initialization fails but
-    does not raise an exception to avoid crashing the application.
-    """
-    try:
-        logger.info("Initializing reranker model: %s", settings.RERANKER_MODEL_NAME)
-        return Reranker(
-            settings.RERANKER_MODEL_NAME,
-            model_type=settings.RERANKER_MODEL_TYPE,
-            api_key=settings.RERANKER_API_KEY,
-        )
-    except Exception as e:  # noqa: BLE001# pylint: disable=broad-exception-caught
-        logger.error("Failed to initialize reranker: %s", str(e))
-        return None
-
-
-def _rerank(reranker: BaseRanker, query: str, original_hits: list[dict]) -> list[dict]:
-    """Rerank the original results using the provided reranker."""
-    docs, doc_ids = prepare_rerank_data(original_hits)
-
-    logger.info("Reranking %d results for query: %s", len(original_hits), query)
-    reranked = reranker.rank(query=query, docs=docs, doc_ids=doc_ids)
-
-    reranked_results: list[dict] = []
-    for reranked_result in reranked.results:
-        matching_hits = [
-            hit for hit in original_hits if hit["_id"] == reranked_result.doc_id
-        ]
-
-        if not matching_hits:
-            logger.warning(
-                "Reranked document ID %s not found in original hits, skipping",
-                reranked_result.doc_id,
-            )
-            continue
-
-        if len(matching_hits) > 1:
-            logger.warning(
-                "Multiple hits found for document ID %s, using first match",
-                reranked_result.doc_id,
-            )
-
-        hit = matching_hits[0]
-        hit["_reranked_score"] = reranked_result.score
-        reranked_results.append(hit)
-
-    logger.info("Reranking completed, returned %d results", len(reranked_results))
-
-    return reranked_results
-
-
-def prepare_rerank_data(original_hits: list[dict]) -> tuple[list[str], list[str]]:
-    """
-    Prepare the documents for reranking by extracting the title and content from the original hits.
-    """
-    docs = []
-    doc_ids = []
-    for hit in original_hits:
-        title = get_language_value(hit["_source"], "title")
-        content = get_language_value(hit["_source"], "content")
-
-        docs.append(format_document(title, content))
-        doc_ids.append(hit["_id"])
-
-    return docs, doc_ids
-
-
-def should_rerank(is_rerank_requested: bool | None) -> bool:
-    """
-    Determine whether to perform reranking based on the input parameter and settings.
-    logs warning if reranking was explicitly requested but the reranker is disabled in settings.
-    falls back to settings.RERANKER_ENABLED if is_rerank_requested is None.
-    """
-    if is_rerank_requested and not settings.RERANKER_ENABLED:
-        logger.warning(
-            "Reranking was explicitly requested but the reranker "
-            "is disabled in settings. Reranking skipped."
-        )
-        return False
-
-    if is_rerank_requested is None:
-        return settings.RERANKER_ENABLED
-
-    return is_rerank_requested and settings.RERANKER_ENABLED
@@ -9,7 +9,6 @@ from core.enums import SearchTypeEnum

 from .embedding import embed_text
 from .opensearch import check_hybrid_search_enabled, opensearch_client
-from .reranking import rerank, should_rerank

 logger = logging.getLogger(__name__)

@@ -18,8 +17,6 @@ logger = logging.getLogger(__name__)
 def search(  # noqa : PLR0913
    q,
    nb_results,
-    order_by,
-    order_direction,
    search_indices,
    reach,
    visited,
@@ -28,7 +25,7 @@ def search(  # noqa : PLR0913
    tags,
    search_type,
    path=None,
-    rerank_requested=None,
+    rescore=False,
 ):
    """Perform an OpenSearch search"""
    query = get_query(
@@ -42,7 +39,7 @@ def search(  # noqa : PLR0913
        path=path,
        search_type=search_type,
    )
-    response = opensearch_client().search(  # pylint: disable=unexpected-keyword-arg
+    return opensearch_client().search(  # pylint: disable=unexpected-keyword-arg
        index=",".join(search_indices),
        body={
            "_source": enums.SOURCE_FIELDS,  # limit the fields to return
@@ -50,13 +47,9 @@ def search(  # noqa : PLR0913
                "number_of_users": {"script": {"source": "doc['users'].size()"}},
                "number_of_groups": {"script": {"source": "doc['groups'].size()"}},
            },
-            "sort": get_sort(
-                query_keys=query.keys(),
-                order_by=order_by,
-                order_direction=order_direction,
-            ),
            "size": nb_results,
            "query": query,
+            "rescore": get_rescore(nb_results=nb_results) if rescore else [],
        },
        params=get_params(query_keys=query.keys()),
        # disable=unexpected-keyword-arg because
@@ -64,14 +57,6 @@ def search(  # noqa : PLR0913
        ignore_unavailable=True,
    )

-    if should_rerank(rerank_requested) and q != "*":
-        response["hits"]["hits"] = rerank(
-            query=q,
-            hits=response["hits"]["hits"],
-        )
-
-    return response
-

 # pylint: disable=too-many-arguments, too-many-positional-arguments
 def get_query(  # noqa : PLR0913
@@ -237,17 +222,35 @@ def get_filter(  # noqa : PLR0913
    return filters


-def get_sort(query_keys, order_by, order_direction):
-    """Build OpenSearch sort clause"""
-    # Add sorting logic based on relevance or specified field
-    if "hybrid" in query_keys:
-        # sorting by other field than "_score" is not supported in hybrid search
-        # see: https://github.com/opensearch-project/neural-search/issues/866
-        return {"_score": {"order": order_direction}}
-    if order_by == enums.RELEVANCE:
-        return {"_score": {"order": order_direction}}
-
-    return {order_by: {"order": order_direction}}
+def get_rescore(nb_results):
+    """
+    Build rescore query.
+    Rescore is based on the `updated_at` field to boost more recently updated documents
+    """
+    return [
+        {
+            "window_size": nb_results,
+            "query": {
+                "rescore_query_weight": settings.RESCORE_UPDATED_AT_WEIGHT,
+                "rescore_query": {
+                    "function_score": {
+                        "functions": [
+                            {
+                                "gauss": {
+                                    "updated_at": {
+                                        "origin": "now",
+                                        "offset": settings.RESCORE_UPDATED_AT_OFFSET,
+                                        "scale": settings.RESCORE_UPDATED_AT_SCALE,
+                                        "decay": settings.RESCORE_UPDATED_AT_DECAY,
+                                    }
+                                }
+                            }
+                        ],
+                    }
+                },
+            },
+        }
+    ]


 def get_params(query_keys):
@@ -1,6 +1,7 @@
 """Tests indexing documents in OpenSearch over the API"""

 import datetime
+import logging

 from django.utils import timezone

@@ -118,6 +119,43 @@ def test_api_documents_index_single_hybrid_enabled_success(settings):
        assert len(chunk["content"]) < len(document["content"])


+@responses.activate
+def test_api_documents_index_with_wrong_embedding_dimension(settings, caplog):
+    """Test embedding with wrong dimension should log a warning and not index the embedding."""
+    service = factories.ServiceFactory()
+    enable_hybrid_search(settings)
+    settings.EMBEDDING_DIMENSION = 8
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+
+    document = factories.DocumentSchemaFactory.build()
+
+    with caplog.at_level(logging.WARNING):
+        APIClient().post(
+            "/api/v1.0/documents/index/",
+            document,
+            HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
+            format="json",
+        )
+
+    assert any(
+        "unexpected embedding dimension: EMBEDDING_DIMENSION is set to 8 "
+        "but the configured embedding model returned a vector of dimension 1024"
+        in message
+        for message in caplog.messages
+    )
+
+    new_indexed_document = opensearch.opensearch_client().get(
+        index=service.index_name, id=str(document["id"])
+    )
+    # check embedding
+    assert new_indexed_document["_source"]["chunks"] is None
+
+
 def test_api_documents_index_language_params():
    """language_code query param should control which language is indexed."""
    service = factories.ServiceFactory()
@@ -6,9 +6,7 @@ of documents is slow and better done only once.
 """

 import logging
-import operator
 import random
-from unittest import mock

 import pytest
 import responses
@@ -195,7 +193,11 @@ def test_api_documents_search_match_all(settings):

    response = APIClient().post(
        "/api/v1.0/documents/search/",
-        {"q": "*", "visited": [doc["id"] for doc in documents]},
+        {
+            "q": "*",
+            "visited": [doc["id"] for doc in documents],
+            "rescore": False,
+        },
        format="json",
        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
    )
@@ -500,181 +502,6 @@ def test_api_documents_hybrid_search(settings):
    }


-@responses.activate
-def test_api_documents_search_ordering_by_fields(settings):
-    """It should be possible to order by several fields"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        4, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    parameters = [
-        (enums.CREATED_AT, "asc"),
-        (enums.CREATED_AT, "desc"),
-        (enums.UPDATED_AT, "asc"),
-        (enums.UPDATED_AT, "desc"),
-        (enums.SIZE, "asc"),
-        (enums.SIZE, "desc"),
-        (enums.REACH, "asc"),
-        (enums.REACH, "desc"),
-    ]
-
-    for field, direction in parameters:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": field,
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-        assert response.status_code == 200
-        data = response.json()
-        assert len(data) == 4
-
-        # Check that results are sorted by the field as expected
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(data) - 1):
-            assert compare(data[i]["_source"][field], data[i + 1]["_source"][field])
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_relevance(settings):
-    """It should be possible to order by relevance (score)"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        4, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    for direction in ["asc", "desc"]:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": "relevance",
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-        assert response.status_code == 200
-        data = response.json()
-        assert len(data) == 4
-
-        # Check that results are sorted by score as expected
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(data) - 1):
-            assert compare(data[i]["_score"], data[i + 1]["_score"])
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_unknown_field(settings):
-    """Trying to sort by an unknown field should return a 400 error"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    # Setup: Initialize the service and documents only once
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        2, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    # Define the parameters manually
-    directions = ["asc", "desc"]
-
-    # Perform the parameterized tests
-    for direction in directions:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": "unknown",
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-        assert response.status_code == 400
-        assert response.json() == [
-            {
-                "loc": ["order_by"],
-                "msg": (
-                    "Input should be 'relevance', 'title', 'created_at', "
-                    "'updated_at', 'size' or 'reach'"
-                ),
-                "type": "literal_error",
-            }
-        ]
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_unknown_direction(settings):
-    """Trying to sort with an unknown direction should return a 400 error"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        2, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    for field in enums.ORDER_BY_OPTIONS:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": field,
-                "order_direction": "unknown",
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-        assert response.status_code == 400
-        assert response.json() == [
-            {
-                "loc": ["order_direction"],
-                "msg": "Input should be 'asc' or 'desc'",
-                "type": "literal_error",
-            }
-        ]
-
-
@responses.activate
 def test_api_documents_search_filtering_by_reach(settings):
    """It should be possible to filter results by their reach"""
@@ -734,6 +561,7 @@ def test_api_documents_search_with_nb_results(settings):
            "q": "*",
            "nb_results": nb_results,
            "visited": [doc["id"] for doc in documents],
+            "rescore": False,
        },
        format="json",
        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
@@ -756,7 +584,7 @@ def test_api_documents_search_with_nb_results(settings):
    )
    assert response.status_code == 200
    data = response.json()
-    assert [r["_id"] for r in data] == ids[0:nb_results]
+    assert {r["_id"] for r in data} == set(ids[0:nb_results])

    nb_results = 10
    response = APIClient().post(
@@ -772,7 +600,7 @@ def test_api_documents_search_with_nb_results(settings):
    assert response.status_code == 200
    data = response.json()
    # nb_results > total number of documents => returns all documents
-    assert [r["_id"] for r in data] == ids[0:9]
+    assert {r["_id"] for r in data} == set(ids[0:9])


@responses.activate
@@ -841,12 +669,13 @@ def test_api_documents_search_nb_results_with_filtering(settings):
            "reach": "public",
            "nb_results": nb_results,
            "visited": public_ids,
+            "rescore_enable": False,
        },
        format="json",
        HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
    )
    assert response.status_code == 200
-    assert [r["_id"] for r in response.json()] == public_ids[0:nb_results]
+    assert {r["_id"] for r in response.json()} == set(public_ids[0:nb_results])


@responses.activate
@@ -994,36 +823,3 @@ def test_api_documents_search_filtering_by_path(settings):
    assert len(response.json()) == 2
    for hit in response.json():
        assert hit["_source"]["path"].startswith(path_filter)
-
-
-@responses.activate
-@mock.patch("core.services.reranking.rerank")
-def test_api_documents_rerank_blocked(
-    mock_rerank,
-    settings,
-    caplog,
-):
-    """Test can disable rerank at run time even when enabled in settings"""
-    settings.RERANKER_ENABLED = True
-
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    service = factories.ServiceFactory()
-
-    documents = bulk_create_documents([{"title": "Fox"}, {"title": "Wolf"}])
-    prepare_index(service.index_name, documents)
-    mock_rerank.return_value = documents
-
-    with caplog.at_level(logging.INFO):
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "programming",
-                "rerank": False,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-    assert response.status_code == 200
-    assert not mock_rerank.called
@@ -1,140 +0,0 @@
-"""
-Test suite for reranking service
-"""
-
-import logging
-from unittest.mock import MagicMock, patch
-
-import pytest
-
-from core.services.reranking import rerank
-
-pytestmark = pytest.mark.django_db
-
-logger = logging.getLogger(__name__)
-
-
-ORIGINAL_HITS = [
-    {
-        "_id": "doc-1",
-        "_score": 1.5,
-        "_source": {
-            "title.en": "Dogs and wolves",
-            "content.en": "Dogs are domesticated wolves",
-        },
-    },
-    {
-        "_id": "doc-2",
-        "_score": 1.2,
-        "_source": {
-            "title.en": "Cats as pets",
-            "content.en": "Cats are popular pets",
-        },
-    },
-    {
-        "_id": "doc-3",
-        "_score": 1.0,
-        "_source": {
-            "title.en": "Birds in nature",
-            "content.en": "Birds fly in the sky",
-        },
-    },
-]
-
-RERANKER_RESULTS = [
-    {"doc_id": "doc-2", "score": 0.95},
-    {"doc_id": "doc-1", "score": 0.87},
-    {"doc_id": "doc-3", "score": 0.45},
-]
-
-
-@pytest.fixture(name="get_reranker")
-def mock_get_reranker():
-    """Mock reranker result with reranked scores"""
-    with patch("core.services.reranking.get_reranker") as mocked_get_reranker:
-        reranker_result = MagicMock()
-
-        reranker_result.results = [
-            MagicMock(**reranker_result) for reranker_result in RERANKER_RESULTS
-        ]
-
-        mock_reranker = MagicMock()
-        mock_reranker.rank.return_value = reranker_result
-        mocked_get_reranker.return_value = mock_reranker
-
-        # Yield both the patched function and the mock reranker instance
-        yield mock_reranker
-
-
-@patch("core.services.reranking.get_reranker")
-def test_return_original_results_if_reranker_import_fails(mocked_get_reranker, caplog):
-    """Test that original results are returned if reranker import fails"""
-    mocked_get_reranker.return_value = None
-
-    with caplog.at_level(logging.WARNING):
-        result = rerank("test query", ORIGINAL_HITS)
-
-    assert result == ORIGINAL_HITS
-    assert any(
-        "Could not import reranker, returning original results" in message
-        for message in caplog.messages
-    )
-
-
-@patch("core.services.reranking.get_reranker")
-def test_return_original_results_if_reranking_fails(mocked_get_reranker, caplog):
-    """Test that original results are returned if reranking fails"""
-    mock_reranker = MagicMock()
-    mock_reranker.rank.side_effect = Exception("Reranking service error")
-    mocked_get_reranker.return_value = mock_reranker
-
-    q = "test query"
-    with caplog.at_level(logging.ERROR):
-        result = rerank(q, ORIGINAL_HITS)
-
-    assert result == ORIGINAL_HITS
-    assert any(
-        "Reranking failed: Reranking service error, returning original results"
-        in message
-        for message in caplog.messages
-    )
-
-
-def test_reranking_success(get_reranker, caplog):
-    """Test successful reranking of search results"""
-    mock_reranker = get_reranker
-
-    q = "test query"
-    with caplog.at_level(logging.INFO):
-        reranked_hits = rerank(q, ORIGINAL_HITS.copy())
-
-    assert any(
-        f"Reranking 3 results for query: {q}" in message for message in caplog.messages
-    )
-    assert any(
-        "Reranking completed, returned 3 results" in message
-        for message in caplog.messages
-    )
-
-    mock_reranker.rank.assert_called_once()
-    call_args = mock_reranker.rank.call_args
-    assert call_args.kwargs["query"] == q
-    assert len(call_args.kwargs["docs"]) == len(RERANKER_RESULTS)
-
-    for ranked_hit_index, ranked_hit in enumerate(reranked_hits):
-        reranker_result = RERANKER_RESULTS[ranked_hit_index]
-        expected_doc_id = reranker_result["doc_id"]
-        expected_score = reranker_result["score"]
-        expected_reranked_hit = ORIGINAL_HITS[
-            next(
-                _hit_index
-                for _hit_index, hit in enumerate(ORIGINAL_HITS)
-                if hit["_id"] == expected_doc_id
-            )
-        ]
-
-        assert expected_doc_id == ranked_hit["_id"]
-        assert expected_reranked_hit["_id"] == ranked_hit["_id"]
-        assert expected_reranked_hit["_source"] == ranked_hit["_source"]
-        assert expected_reranked_hit["_score"] == ranked_hit["_score"]
-        assert expected_score == expected_reranked_hit["_reranked_score"]
@@ -2,10 +2,9 @@
 Test suite for opensearch search service
 """

+import datetime
 import logging
-import operator
 from json import dumps as json_dumps
-from unittest.mock import patch

 import pytest
 import responses
@@ -33,8 +32,6 @@ def search_params(service):
    """Build opensearch.search() parameters for tests using the service index name"""
    return {
        "nb_results": 20,
-        "order_by": "relevance",
-        "order_direction": "desc",
        "search_indices": {service.index_name},
        "reach": None,
        "user_sub": "user_sub",
@@ -408,41 +405,6 @@ def test_match_all(settings, caplog):
    assert len(result["hits"]["hits"]) == 3


-@responses.activate
-def test_search_ordering_by_relevance(settings, caplog):
-    """Test the hybrid supports ordering by relevance asc and desc"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    q = "canine pet"
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    for direction in ["asc", "desc"]:
-        with caplog.at_level(logging.INFO):
-            result = search(
-                q=q, **{**search_params(service), "order_direction": direction}
-            )
-
-        # Check that results are sorted by score as expected
-        hits = result["hits"]["hits"]
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(hits) - 1):
-            assert compare(hits[i]["_score"], hits[i + 1]["_score"])
-
-
@responses.activate
 def test_hybrid_search_number_of_matches(settings):
    """
@@ -715,56 +677,41 @@ def test_search_filtering_by_query_path_and_tag():
    assert returned_ids == expected_ids


-# pylint: disable=too-many-arguments, too-many-positional-arguments
-@responses.activate
-@patch("core.services.search.rerank")
-@pytest.mark.parametrize(
-    "rerank_requested, reranker_enabled, rerank_called, caplog_messages",
-    [
-        (True, True, True, []),
-        (
-            True,
-            False,
-            False,
-            [
-                "Reranking was explicitly requested but the reranker is "
-                "disabled in settings. Reranking skipped."
-            ],
-        ),
-        (False, True, False, []),
-        (False, False, False, []),
-        (None, False, False, []),
-        (None, True, True, []),
-    ],
-)
-def test_search_rerank(  # noqa : PLR0913
-    mock_rerank,
-    rerank_requested,
-    reranker_enabled,
-    rerank_called,
-    caplog_messages,
-    settings,
-    caplog,
-):
-    """Test that rerank activation according to args and settings"""
-    settings.RERANKER_ENABLED = reranker_enabled
-
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    mock_rerank.return_value = documents
+def test_search_with_rescore(settings):
+    """Test rescore feature"""
    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    today = datetime.datetime.today()
+    forty_days_ago = today - datetime.timedelta(days=40)
+    documents = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="my first document",
+            created_at=today,
+            updated_at=today,
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="another document",
+            created_at=forty_days_ago,
+            updated_at=forty_days_ago,
+        ),
+    ]
    prepare_index(service.index_name, documents)

-    with caplog.at_level(logging.INFO):
-        search(
-            q="canine pet", rerank_requested=rerank_requested, **search_params(service)
-        )
+    # set a cray big RESCORE_UPDATED_AT_WEIGHT to demonstrate the effect of boosting on rescores
+    settings.RESCORE_UPDATED_AT_WEIGHT = 200

-    assert mock_rerank.called == rerank_called
-    for message in caplog_messages:
-        assert message in caplog.text
+    results = search(
+        q="another document",
+        **{
+            **search_params(service),
+            "rescore": True,
+        },
+    )
+
+    hits = results["hits"]["hits"]
+    # the first document is ranked first because it more recent
+    # even though the second one matches the query better
+    assert hits[0]["_source"]["title.en"] == "my first document"
+    assert hits[1]["_source"]["title.en"] == "another document"
@@ -332,12 +332,6 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
        path : str, optional
            Filter results based on the 'path' field. Only documents whose path
            starts with the provided value will be returned.
-        order_by : str, optional
-            Order results by 'relevance', 'created_at', 'updated_at', or 'size'.
-            Defaults to 'relevance' if not specified.
-        order_direction : str, optional
-            Order direction, 'asc' for ascending or 'desc' for descending.
-            Defaults to 'desc'.
        nb_results : int, optional
            The number of results to return.
            Defaults to 50 if not specified.
@@ -354,9 +348,6 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
            - 'full_text': Uses only full-text search, even if hybrid search is enabled
                on the server.
            if the not specified, the server will use hybrid search when enabled
-        rerank : bool, optional
-            Enable or disable reranking of results. If not specified, falls back to
-            the RERANKER_ENABLED setting.

        Returns:
        --------
@@ -367,6 +358,7 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
        # Get list of groups related to the user from SCIM provider (consider caching result)
        audience = self._get_service_provider_audience()
        user_sub = self.request.user.sub
+        groups = []
        params = schemas.SearchQueryParametersSchema(**request.data)

        # Get index list for search query
@@ -383,13 +375,11 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
        result = search(
            q=params.q,
            nb_results=params.nb_results,
-            order_by=params.order_by,
-            order_direction=params.order_direction,
            search_indices=search_indices,
            reach=params.reach,
            visited=params.visited,
            user_sub=user_sub,
-            groups=[],
+            groups=groups,
            tags=params.tags,
            path=params.path,
            search_type=params.search_type
@@ -397,7 +387,7 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
            else SearchTypeEnum.HYBRID
            if check_hybrid_search_enabled()
            else SearchTypeEnum.FULL_TEXT,
-            rerank_requested=params.rerank,
+            rescore=params.rescore,
        )["hits"]["hits"]
        logger.info("found %d results", len(result))
        logger.debug("results %s", result)
@@ -38,8 +38,6 @@ class Command(BaseCommand):
    index_name = "evaluation-index"
    search_params = {
        "nb_results": 20,
-        "order_by": "relevance",
-        "order_direction": "desc",
        "search_indices": {index_name},
        "reach": None,
        "user_sub": "user_sub",
@@ -318,23 +318,18 @@ class Base(Configuration):
    EMBEDDING_DIMENSION = values.IntegerValue(
        default=1024, environ_name="EMBEDDING_DIMENSION", environ_prefix=None
    )
-
-    # Reranking
-    RERANKER_ENABLED = values.BooleanValue(
-        default=False, environ_name="RERANKER_ENABLED", environ_prefix=None
+    # rescore
+    RESCORE_UPDATED_AT_WEIGHT = values.FloatValue(
+        default=0.2, environ_name="RESCORE_UPDATED_AT_WEIGHT", environ_prefix=None
    )
-    RERANKER_MODEL_NAME = values.Value(
-        default=None,
-        environ_name="RERANKER_MODEL_NAME",
-        environ_prefix=None,
+    RESCORE_UPDATED_AT_OFFSET = values.Value(
+        default="2d", environ_name="RESCORE_UPDATED_AT_OFFSET", environ_prefix=None
    )
-    RERANKER_MODEL_TYPE = values.Value(
-        default=None,
-        environ_name="RERANKER_MODEL_TYPE",
-        environ_prefix=None,
+    RESCORE_UPDATED_AT_SCALE = values.Value(
+        default="6d", environ_name="RESCORE_UPDATED_AT_SCALE", environ_prefix=None
    )
-    RERANKER_API_KEY = values.Value(
-        default=None, environ_name="RERANKER_API_KEY", environ_prefix=None
+    RESCORE_UPDATED_AT_DECAY = values.IntegerValue(
+        default=0.5, environ_name="RESCORE_UPDATED_AT_SCALE", environ_prefix=None
    )

    # CORS
@@ -44,7 +44,6 @@ dependencies = [
    "pydantic==2.12.5",
    "pyjwt==2.10.1",
    "requests==2.32.5",
-    "rerankers[api]==0.10.0",
    "sentry-sdk==2.48.0",
    "url-normalize==2.2.1",
    "opensearch-py==3.1.0",
@@ -514,7 +514,6 @@ dependencies = [
    { name = "pyjwt" },
    { name = "redis" },
    { name = "requests" },
-    { name = "rerankers", extra = ["api"] },
    { name = "sentry-sdk" },
    { name = "url-normalize" },
    { name = "whitenoise" },
@@ -575,7 +574,6 @@ requires-dist = [
    { name = "pytest-xdist", marker = "extra == 'dev'", specifier = "==3.8.0" },
    { name = "redis", specifier = "==5.3.1" },
    { name = "requests", specifier = "==2.32.5" },
-    { name = "rerankers", extras = ["api"], specifier = "==0.10.0" },
    { name = "responses", marker = "extra == 'dev'", specifier = "==0.25.8" },
    { name = "ruff", marker = "extra == 'dev'", specifier = "==0.14.10" },
    { name = "sentry-sdk", specifier = "==2.48.0" },
@@ -1419,20 +1417,6 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" },
 ]

-[[package]]
-name = "rerankers"
-version = "0.10.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a2/1e/3ed2026be7c135939905eac4f50d1bf8339180821c6757b2e91b83de2fa5/rerankers-0.10.0.tar.gz", hash = "sha256:b8e8b363abc4e9757151956949c27b197993c0a774437287a932f855afc17a73", size = 49679, upload-time = "2025-05-22T08:22:53.396Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/df/ed/f3b81ca8743d69b95d679b95e6e1d22cb7cc678ae77c6a57827303a7e48c/rerankers-0.10.0-py3-none-any.whl", hash = "sha256:634a6befa130a245ed46022ade217ee482869448f01aae2051ed54d7d5bd2791", size = 53084, upload-time = "2025-05-22T08:22:52.022Z" },
-]
-
-[package.optional-dependencies]
-api = [
-    { name = "requests" },
-]
-
 [[package]]
 name = "responses"
 version = "0.25.8"
Author	SHA1	Message	Date
charles	efe12bcefc	✅(backend) test i am testing the rescore feature	2026-03-16 09:51:58 +01:00
charles	bed2bd5203	✨(backend) add enable_rescore params I am making rescore optional.	2026-03-16 09:51:58 +01:00
charles	bd5b5f740f	⚰️(backend) remove sorting sorting is not supported with hybrid search. I should have removed it long ago. It is not supported also with rerank.	2026-03-16 09:51:58 +01:00
charles	997e6f3545	✨(backend) rescore documents on updated_at We hypothesised that users will be interested in most recently updated documents. To improve search results given this hypothesis we want to boost the score of documents based on the `updated_at` field. This commit applies a gaussian boost on the updated_at field	2026-03-16 09:51:58 +01:00
charles	70015e8a7d	🚨(backend) remove dimensions params in embed_text the params dimension is either ignored or crashes the api call depending on the provider	2026-03-16 09:51:58 +01:00
charles	dd197e1f1e	♻️(backend) reorganize evaluation data I am improving the data management of evaluation system	2026-03-10 15:05:14 +01:00