✅(backend) test

i am testing the rescore feature

CHANGELOG.md

@@ -9,7 +9,7 @@ and this project adheres to
 # Unreleased
 
 ## Added
-
+- ✨(backend) add rescore on `updated_at`
 - 👷(docker) add arm64 platform support for image builds
 - ✨(backend) add semantic search
 - ✨(backend) add multi-embedding and chunking
@@ -42,3 +42,6 @@ and this project adheres to
 
 - 🐛(backend) fix missing index creation in 'index/' view
 - 🐛(backend) fix parallel test execution issues
+
+## Removed
+- 🗑️(backend) remove sorting

src/backend/core/enums.py

@@ -38,9 +38,6 @@ UPDATED_AT = "updated_at"
 USERS = "users"
 GROUPS = "groups"
 
-RELEVANCE = "relevance"
-
-ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, REACH)
 SOURCE_FIELDS = (
     TITLE,
     CONTENT,

src/backend/core/schemas.py

@@ -1,6 +1,6 @@
 """Pydantic model to validate documents before indexation."""
 
-from typing import Annotated, List, Literal, Optional
+from typing import Annotated, List, Optional
 
 from django.utils import timezone
 from django.utils.text import slugify
@@ -116,10 +116,9 @@ class SearchQueryParametersSchema(BaseModel):
     reach: Optional[enums.ReachEnum] = None
     tags: StringListParameter = Field(default_factory=list)
     path: Optional[str] = None
-    order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default=enums.RELEVANCE)
-    order_direction: Optional[Literal["asc", "desc"]] = Field(default="desc")
     nb_results: Optional[conint(ge=1, le=300)] = Field(default=50)
     search_type: Optional[SearchTypeEnum] = Field(default=None)
+    rescore: bool = Field(default=True)
 
 
 class DeleteDocumentsSchema(BaseModel):

src/backend/core/services/embedding.py

@@ -21,7 +21,6 @@ def embed_text(text):
         json={
             "input": text,
             "model": settings.EMBEDDING_API_MODEL_NAME,
-            "dimensions": settings.EMBEDDING_DIMENSION,
             "encoding_format": "float",
         },
         timeout=settings.EMBEDDING_REQUEST_TIMEOUT,
@@ -39,4 +38,14 @@ def embed_text(text):
         logger.warning("unexpected embedding response format: %s", response.text)
         return None
 
+    if len(embedding) != settings.EMBEDDING_DIMENSION:
+        logger.warning(
+            "unexpected embedding dimension: "
+            "EMBEDDING_DIMENSION is set to %d "
+            "but the configured embedding model returned a vector of dimension %d",
+            settings.EMBEDDING_DIMENSION,
+            len(embedding),
+        )
+        return None
+
     return embedding

src/backend/core/services/search.py

@@ -17,8 +17,6 @@ logger = logging.getLogger(__name__)
 def search(  # noqa : PLR0913
     q,
     nb_results,
-    order_by,
-    order_direction,
     search_indices,
     reach,
     visited,
@@ -27,6 +25,7 @@ def search(  # noqa : PLR0913
     tags,
     search_type,
     path=None,
+    rescore=False,
 ):
     """Perform an OpenSearch search"""
     query = get_query(
@@ -48,13 +47,9 @@ def search(  # noqa : PLR0913
                 "number_of_users": {"script": {"source": "doc['users'].size()"}},
                 "number_of_groups": {"script": {"source": "doc['groups'].size()"}},
             },
-            "sort": get_sort(
-                query_keys=query.keys(),
-                order_by=order_by,
-                order_direction=order_direction,
-            ),
             "size": nb_results,
             "query": query,
+            "rescore": get_rescore(nb_results=nb_results) if rescore else [],
         },
         params=get_params(query_keys=query.keys()),
         # disable=unexpected-keyword-arg because
@@ -227,17 +222,35 @@ def get_filter(  # noqa : PLR0913
     return filters
 
 
-def get_sort(query_keys, order_by, order_direction):
-    """Build OpenSearch sort clause"""
-    # Add sorting logic based on relevance or specified field
-    if "hybrid" in query_keys:
-        # sorting by other field than "_score" is not supported in hybrid search
-        # see: https://github.com/opensearch-project/neural-search/issues/866
-        return {"_score": {"order": order_direction}}
-    if order_by == enums.RELEVANCE:
-        return {"_score": {"order": order_direction}}
-
-    return {order_by: {"order": order_direction}}
+def get_rescore(nb_results):
+    """
+    Build rescore query.
+    Rescore is based on the `updated_at` field to boost more recently updated documents
+    """
+    return [
+        {
+            "window_size": nb_results,
+            "query": {
+                "rescore_query_weight": settings.RESCORE_UPDATED_AT_WEIGHT,
+                "rescore_query": {
+                    "function_score": {
+                        "functions": [
+                            {
+                                "gauss": {
+                                    "updated_at": {
+                                        "origin": "now",
+                                        "offset": settings.RESCORE_UPDATED_AT_OFFSET,
+                                        "scale": settings.RESCORE_UPDATED_AT_SCALE,
+                                        "decay": settings.RESCORE_UPDATED_AT_DECAY,
+                                    }
+                                }
+                            }
+                        ],
+                    }
+                },
+            },
+        }
+    ]
 
 
 def get_params(query_keys):

src/backend/core/tests/test_api_documents_index_single.py

@@ -1,6 +1,7 @@
 """Tests indexing documents in OpenSearch over the API"""
 
 import datetime
+import logging
 
 from django.utils import timezone
 
@@ -118,6 +119,43 @@ def test_api_documents_index_single_hybrid_enabled_success(settings):
         assert len(chunk["content"]) < len(document["content"])
 
 
+@responses.activate
+def test_api_documents_index_with_wrong_embedding_dimension(settings, caplog):
+    """Test embedding with wrong dimension should log a warning and not index the embedding."""
+    service = factories.ServiceFactory()
+    enable_hybrid_search(settings)
+    settings.EMBEDDING_DIMENSION = 8
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+
+    document = factories.DocumentSchemaFactory.build()
+
+    with caplog.at_level(logging.WARNING):
+        APIClient().post(
+            "/api/v1.0/documents/index/",
+            document,
+            HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
+            format="json",
+        )
+
+    assert any(
+        "unexpected embedding dimension: EMBEDDING_DIMENSION is set to 8 "
+        "but the configured embedding model returned a vector of dimension 1024"
+        in message
+        for message in caplog.messages
+    )
+
+    new_indexed_document = opensearch.opensearch_client().get(
+        index=service.index_name, id=str(document["id"])
+    )
+    # check embedding
+    assert new_indexed_document["_source"]["chunks"] is None
+
+
 def test_api_documents_index_language_params():
     """language_code query param should control which language is indexed."""
     service = factories.ServiceFactory()

src/backend/core/tests/test_api_documents_search.py

@@ -6,7 +6,6 @@ of documents is slow and better done only once.
 """
 
 import logging
-import operator
 import random
 
 import pytest
@@ -194,7 +193,11 @@ def test_api_documents_search_match_all(settings):
 
     response = APIClient().post(
         "/api/v1.0/documents/search/",
-        {"q": "*", "visited": [doc["id"] for doc in documents]},
+        {
+            "q": "*",
+            "visited": [doc["id"] for doc in documents],
+            "rescore": False,
+        },
         format="json",
         HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
@@ -499,181 +502,6 @@ def test_api_documents_hybrid_search(settings):
     }
 
 
-@responses.activate
-def test_api_documents_search_ordering_by_fields(settings):
-    """It should be possible to order by several fields"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        4, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    parameters = [
-        (enums.CREATED_AT, "asc"),
-        (enums.CREATED_AT, "desc"),
-        (enums.UPDATED_AT, "asc"),
-        (enums.UPDATED_AT, "desc"),
-        (enums.SIZE, "asc"),
-        (enums.SIZE, "desc"),
-        (enums.REACH, "asc"),
-        (enums.REACH, "desc"),
-    ]
-
-    for field, direction in parameters:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": field,
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-        assert response.status_code == 200
-        data = response.json()
-        assert len(data) == 4
-
-        # Check that results are sorted by the field as expected
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(data) - 1):
-            assert compare(data[i]["_source"][field], data[i + 1]["_source"][field])
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_relevance(settings):
-    """It should be possible to order by relevance (score)"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        4, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    for direction in ["asc", "desc"]:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": "relevance",
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-        assert response.status_code == 200
-        data = response.json()
-        assert len(data) == 4
-
-        # Check that results are sorted by score as expected
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(data) - 1):
-            assert compare(data[i]["_score"], data[i + 1]["_score"])
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_unknown_field(settings):
-    """Trying to sort by an unknown field should return a 400 error"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    # Setup: Initialize the service and documents only once
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        2, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    # Define the parameters manually
-    directions = ["asc", "desc"]
-
-    # Perform the parameterized tests
-    for direction in directions:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": "unknown",
-                "order_direction": direction,
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-        assert response.status_code == 400
-        assert response.json() == [
-            {
-                "loc": ["order_by"],
-                "msg": (
-                    "Input should be 'relevance', 'title', 'created_at', "
-                    "'updated_at', 'size' or 'reach'"
-                ),
-                "type": "literal_error",
-            }
-        ]
-
-
-@responses.activate
-def test_api_documents_search_ordering_by_unknown_direction(settings):
-    """Trying to sort with an unknown direction should return a 400 error"""
-    setup_oicd_resource_server(responses, settings, sub="user_sub")
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-    service = factories.ServiceFactory()
-    documents = factories.DocumentSchemaFactory.build_batch(
-        2, reach=random.choice(["public", "authenticated"])
-    )
-    prepare_index(service.index_name, documents)
-
-    for field in enums.ORDER_BY_OPTIONS:
-        response = APIClient().post(
-            "/api/v1.0/documents/search/",
-            {
-                "q": "*",
-                "order_by": field,
-                "order_direction": "unknown",
-                "visited": [doc["id"] for doc in documents],
-            },
-            format="json",
-            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
-        )
-
-        assert response.status_code == 400
-        assert response.json() == [
-            {
-                "loc": ["order_direction"],
-                "msg": "Input should be 'asc' or 'desc'",
-                "type": "literal_error",
-            }
-        ]
-
-
 @responses.activate
 def test_api_documents_search_filtering_by_reach(settings):
     """It should be possible to filter results by their reach"""
@@ -733,6 +561,7 @@ def test_api_documents_search_with_nb_results(settings):
             "q": "*",
             "nb_results": nb_results,
             "visited": [doc["id"] for doc in documents],
+            "rescore": False,
         },
         format="json",
         HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
@@ -755,7 +584,7 @@ def test_api_documents_search_with_nb_results(settings):
     )
     assert response.status_code == 200
     data = response.json()
-    assert [r["_id"] for r in data] == ids[0:nb_results]
+    assert {r["_id"] for r in data} == set(ids[0:nb_results])
 
     nb_results = 10
     response = APIClient().post(
@@ -771,7 +600,7 @@ def test_api_documents_search_with_nb_results(settings):
     assert response.status_code == 200
     data = response.json()
     # nb_results > total number of documents => returns all documents
-    assert [r["_id"] for r in data] == ids[0:9]
+    assert {r["_id"] for r in data} == set(ids[0:9])
 
 
 @responses.activate
@@ -840,12 +669,13 @@ def test_api_documents_search_nb_results_with_filtering(settings):
             "reach": "public",
             "nb_results": nb_results,
             "visited": public_ids,
+            "rescore_enable": False,
         },
         format="json",
         HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
     assert response.status_code == 200
-    assert [r["_id"] for r in response.json()] == public_ids[0:nb_results]
+    assert {r["_id"] for r in response.json()} == set(public_ids[0:nb_results])
 
 
 @responses.activate

src/backend/core/tests/test_search.py

@@ -2,8 +2,8 @@
 Test suite for opensearch search service
 """
 
+import datetime
 import logging
-import operator
 from json import dumps as json_dumps
 
 import pytest
@@ -32,8 +32,6 @@ def search_params(service):
     """Build opensearch.search() parameters for tests using the service index name"""
     return {
         "nb_results": 20,
-        "order_by": "relevance",
-        "order_direction": "desc",
         "search_indices": {service.index_name},
         "reach": None,
         "user_sub": "user_sub",
@@ -407,41 +405,6 @@ def test_match_all(settings, caplog):
     assert len(result["hits"]["hits"]) == 3
 
 
-@responses.activate
-def test_search_ordering_by_relevance(settings, caplog):
-    """Test the hybrid supports ordering by relevance asc and desc"""
-    enable_hybrid_search(settings)
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    documents = bulk_create_documents(
-        [
-            {"title": "wolf", "content": "wolves live in packs and hunt together"},
-            {"title": "dog", "content": "dogs are loyal domestic animals"},
-            {"title": "cat", "content": "cats are curious and independent pets"},
-        ]
-    )
-    q = "canine pet"
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-    prepare_index(service.index_name, documents)
-
-    for direction in ["asc", "desc"]:
-        with caplog.at_level(logging.INFO):
-            result = search(
-                q=q, **{**search_params(service), "order_direction": direction}
-            )
-
-        # Check that results are sorted by score as expected
-        hits = result["hits"]["hits"]
-        compare = operator.le if direction == "asc" else operator.ge
-        for i in range(len(hits) - 1):
-            assert compare(hits[i]["_score"], hits[i + 1]["_score"])
-
-
 @responses.activate
 def test_hybrid_search_number_of_matches(settings):
     """
@@ -712,3 +675,43 @@ def test_search_filtering_by_query_path_and_tag():
 
     assert result["hits"]["total"]["value"] == len(documents_to_search)
     assert returned_ids == expected_ids
+
+
+def test_search_with_rescore(settings):
+    """Test rescore feature"""
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+
+    today = datetime.datetime.today()
+    forty_days_ago = today - datetime.timedelta(days=40)
+    documents = [
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="my first document",
+            created_at=today,
+            updated_at=today,
+        ),
+        factories.DocumentSchemaFactory.build(
+            users=["user_sub"],
+            title="another document",
+            created_at=forty_days_ago,
+            updated_at=forty_days_ago,
+        ),
+    ]
+    prepare_index(service.index_name, documents)
+
+    # set a cray big RESCORE_UPDATED_AT_WEIGHT to demonstrate the effect of boosting on rescores
+    settings.RESCORE_UPDATED_AT_WEIGHT = 200
+
+    results = search(
+        q="another document",
+        **{
+            **search_params(service),
+            "rescore": True,
+        },
+    )
+
+    hits = results["hits"]["hits"]
+    # the first document is ranked first because it more recent
+    # even though the second one matches the query better
+    assert hits[0]["_source"]["title.en"] == "my first document"
+    assert hits[1]["_source"]["title.en"] == "another document"

src/backend/core/views.py

@@ -332,12 +332,6 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
         path : str, optional
             Filter results based on the 'path' field. Only documents whose path
             starts with the provided value will be returned.
-        order_by : str, optional
-            Order results by 'relevance', 'created_at', 'updated_at', or 'size'.
-            Defaults to 'relevance' if not specified.
-        order_direction : str, optional
-            Order direction, 'asc' for ascending or 'desc' for descending.
-            Defaults to 'desc'.
         nb_results : int, optional
             The number of results to return.
             Defaults to 50 if not specified.
@@ -381,8 +375,6 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
         result = search(
             q=params.q,
             nb_results=params.nb_results,
-            order_by=params.order_by,
-            order_direction=params.order_direction,
             search_indices=search_indices,
             reach=params.reach,
             visited=params.visited,
@@ -395,6 +387,7 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
             else SearchTypeEnum.HYBRID
             if check_hybrid_search_enabled()
             else SearchTypeEnum.FULL_TEXT,
+            rescore=params.rescore,
         )["hits"]["hits"]
         logger.info("found %d results", len(result))
         logger.debug("results %s", result)

src/backend/evaluation/management/commands/evaluate_search_engine.py

@@ -38,8 +38,6 @@ class Command(BaseCommand):
     index_name = "evaluation-index"
     search_params = {
         "nb_results": 20,
-        "order_by": "relevance",
-        "order_direction": "desc",
         "search_indices": {index_name},
         "reach": None,
         "user_sub": "user_sub",

src/backend/find/settings.py

@@ -318,6 +318,19 @@ class Base(Configuration):
     EMBEDDING_DIMENSION = values.IntegerValue(
         default=1024, environ_name="EMBEDDING_DIMENSION", environ_prefix=None
     )
+    # rescore
+    RESCORE_UPDATED_AT_WEIGHT = values.FloatValue(
+        default=0.2, environ_name="RESCORE_UPDATED_AT_WEIGHT", environ_prefix=None
+    )
+    RESCORE_UPDATED_AT_OFFSET = values.Value(
+        default="2d", environ_name="RESCORE_UPDATED_AT_OFFSET", environ_prefix=None
+    )
+    RESCORE_UPDATED_AT_SCALE = values.Value(
+        default="6d", environ_name="RESCORE_UPDATED_AT_SCALE", environ_prefix=None
+    )
+    RESCORE_UPDATED_AT_DECAY = values.IntegerValue(
+        default=0.5, environ_name="RESCORE_UPDATED_AT_SCALE", environ_prefix=None
+    )
 
     # CORS
     CORS_ALLOW_CREDENTIALS = True