♻️(backend) reorganize evaluation data (#59)

I am improving the data management of evaluation system

CHANGELOG.md

@@ -9,7 +9,7 @@ and this project adheres to
 # Unreleased
 
 ## Added
-- ✨(backend) add rescore on `updated_at`
+
 - 👷(docker) add arm64 platform support for image builds
 - ✨(backend) add semantic search
 - ✨(backend) add multi-embedding and chunking
@@ -42,6 +42,3 @@ and this project adheres to
 
 - 🐛(backend) fix missing index creation in 'index/' view
 - 🐛(backend) fix parallel test execution issues
-
-## Removed
-- 🗑️(backend) remove sorting

src/backend/core/enums.py

@@ -38,6 +38,9 @@ UPDATED_AT = "updated_at"
 USERS = "users"
 GROUPS = "groups"
 
+RELEVANCE = "relevance"
+
+ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, REACH)
 SOURCE_FIELDS = (
     TITLE,
     CONTENT,

src/backend/core/schemas.py

@@ -1,6 +1,6 @@
 """Pydantic model to validate documents before indexation."""
 
-from typing import Annotated, List, Optional
+from typing import Annotated, List, Literal, Optional
 
 from django.utils import timezone
 from django.utils.text import slugify
@@ -116,9 +116,10 @@ class SearchQueryParametersSchema(BaseModel):
     reach: Optional[enums.ReachEnum] = None
     tags: StringListParameter = Field(default_factory=list)
     path: Optional[str] = None
+    order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default=enums.RELEVANCE)
+    order_direction: Optional[Literal["asc", "desc"]] = Field(default="desc")
     nb_results: Optional[conint(ge=1, le=300)] = Field(default=50)
     search_type: Optional[SearchTypeEnum] = Field(default=None)
-    rescore: bool = Field(default=True)
 
 
 class DeleteDocumentsSchema(BaseModel):

src/backend/core/services/embedding.py

@@ -21,6 +21,7 @@ def embed_text(text):
         json={
             "input": text,
             "model": settings.EMBEDDING_API_MODEL_NAME,
+            "dimensions": settings.EMBEDDING_DIMENSION,
             "encoding_format": "float",
         },
         timeout=settings.EMBEDDING_REQUEST_TIMEOUT,
@@ -38,14 +39,4 @@ def embed_text(text):
         logger.warning("unexpected embedding response format: %s", response.text)
         return None
 
-    if len(embedding) != settings.EMBEDDING_DIMENSION:
-        logger.warning(
-            "unexpected embedding dimension: "
-            "EMBEDDING_DIMENSION is set to %d "
-            "but the configured embedding model returned a vector of dimension %d",
-            settings.EMBEDDING_DIMENSION,
-            len(embedding),
-        )
-        return None
-
     return embedding

src/backend/core/services/search.py

@@ -17,6 +17,8 @@ logger = logging.getLogger(__name__)
 def search(  # noqa : PLR0913
     q,
     nb_results,
+    order_by,
+    order_direction,
     search_indices,
     reach,
     visited,
@@ -25,7 +27,6 @@ def search(  # noqa : PLR0913
     tags,
     search_type,
     path=None,
-    rescore=False,
 ):
     """Perform an OpenSearch search"""
     query = get_query(
@@ -47,9 +48,13 @@ def search(  # noqa : PLR0913
                 "number_of_users": {"script": {"source": "doc['users'].size()"}},
                 "number_of_groups": {"script": {"source": "doc['groups'].size()"}},
             },
+            "sort": get_sort(
+                query_keys=query.keys(),
+                order_by=order_by,
+                order_direction=order_direction,
+            ),
             "size": nb_results,
             "query": query,
-            "rescore": get_rescore(nb_results=nb_results) if rescore else [],
         },
         params=get_params(query_keys=query.keys()),
         # disable=unexpected-keyword-arg because
@@ -222,35 +227,17 @@ def get_filter(  # noqa : PLR0913
     return filters
 
 
-def get_rescore(nb_results):
-    """
-    Build rescore query.
-    Rescore is based on the `updated_at` field to boost more recently updated documents
-    """
-    return [
-        {
-            "window_size": nb_results,
-            "query": {
-                "rescore_query_weight": settings.RESCORE_UPDATED_AT_WEIGHT,
-                "rescore_query": {
-                    "function_score": {
-                        "functions": [
-                            {
-                                "gauss": {
-                                    "updated_at": {
-                                        "origin": "now",
-                                        "offset": settings.RESCORE_UPDATED_AT_OFFSET,
-                                        "scale": settings.RESCORE_UPDATED_AT_SCALE,
-                                        "decay": settings.RESCORE_UPDATED_AT_DECAY,
-                                    }
-                                }
-                            }
-                        ],
-                    }
-                },
-            },
-        }
-    ]
+def get_sort(query_keys, order_by, order_direction):
+    """Build OpenSearch sort clause"""
+    # Add sorting logic based on relevance or specified field
+    if "hybrid" in query_keys:
+        # sorting by other field than "_score" is not supported in hybrid search
+        # see: https://github.com/opensearch-project/neural-search/issues/866
+        return {"_score": {"order": order_direction}}
+    if order_by == enums.RELEVANCE:
+        return {"_score": {"order": order_direction}}
+
+    return {order_by: {"order": order_direction}}
 
 
 def get_params(query_keys):

src/backend/core/tests/test_api_documents_index_single.py

@@ -1,7 +1,6 @@
 """Tests indexing documents in OpenSearch over the API"""
 
 import datetime
-import logging
 
 from django.utils import timezone
 
@@ -119,43 +118,6 @@ def test_api_documents_index_single_hybrid_enabled_success(settings):
         assert len(chunk["content"]) < len(document["content"])
 
 
-@responses.activate
-def test_api_documents_index_with_wrong_embedding_dimension(settings, caplog):
-    """Test embedding with wrong dimension should log a warning and not index the embedding."""
-    service = factories.ServiceFactory()
-    enable_hybrid_search(settings)
-    settings.EMBEDDING_DIMENSION = 8
-    responses.add(
-        responses.POST,
-        settings.EMBEDDING_API_PATH,
-        json=albert_embedding_response.response,
-        status=200,
-    )
-
-    document = factories.DocumentSchemaFactory.build()
-
-    with caplog.at_level(logging.WARNING):
-        APIClient().post(
-            "/api/v1.0/documents/index/",
-            document,
-            HTTP_AUTHORIZATION=f"Bearer {service.token:s}",
-            format="json",
-        )
-
-    assert any(
-        "unexpected embedding dimension: EMBEDDING_DIMENSION is set to 8 "
-        "but the configured embedding model returned a vector of dimension 1024"
-        in message
-        for message in caplog.messages
-    )
-
-    new_indexed_document = opensearch.opensearch_client().get(
-        index=service.index_name, id=str(document["id"])
-    )
-    # check embedding
-    assert new_indexed_document["_source"]["chunks"] is None
-
-
 def test_api_documents_index_language_params():
     """language_code query param should control which language is indexed."""
     service = factories.ServiceFactory()

src/backend/core/tests/test_api_documents_search.py

@@ -6,6 +6,7 @@ of documents is slow and better done only once.
 """
 
 import logging
+import operator
 import random
 
 import pytest
@@ -193,11 +194,7 @@ def test_api_documents_search_match_all(settings):
 
     response = APIClient().post(
         "/api/v1.0/documents/search/",
-        {
-            "q": "*",
-            "visited": [doc["id"] for doc in documents],
-            "rescore": False,
-        },
+        {"q": "*", "visited": [doc["id"] for doc in documents]},
         format="json",
         HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
@@ -502,6 +499,181 @@ def test_api_documents_hybrid_search(settings):
     }
 
 
+@responses.activate
+def test_api_documents_search_ordering_by_fields(settings):
+    """It should be possible to order by several fields"""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    service = factories.ServiceFactory()
+    documents = factories.DocumentSchemaFactory.build_batch(
+        4, reach=random.choice(["public", "authenticated"])
+    )
+    prepare_index(service.index_name, documents)
+
+    parameters = [
+        (enums.CREATED_AT, "asc"),
+        (enums.CREATED_AT, "desc"),
+        (enums.UPDATED_AT, "asc"),
+        (enums.UPDATED_AT, "desc"),
+        (enums.SIZE, "asc"),
+        (enums.SIZE, "desc"),
+        (enums.REACH, "asc"),
+        (enums.REACH, "desc"),
+    ]
+
+    for field, direction in parameters:
+        response = APIClient().post(
+            "/api/v1.0/documents/search/",
+            {
+                "q": "*",
+                "order_by": field,
+                "order_direction": direction,
+                "visited": [doc["id"] for doc in documents],
+            },
+            format="json",
+            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert len(data) == 4
+
+        # Check that results are sorted by the field as expected
+        compare = operator.le if direction == "asc" else operator.ge
+        for i in range(len(data) - 1):
+            assert compare(data[i]["_source"][field], data[i + 1]["_source"][field])
+
+
+@responses.activate
+def test_api_documents_search_ordering_by_relevance(settings):
+    """It should be possible to order by relevance (score)"""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    service = factories.ServiceFactory()
+    documents = factories.DocumentSchemaFactory.build_batch(
+        4, reach=random.choice(["public", "authenticated"])
+    )
+    prepare_index(service.index_name, documents)
+
+    for direction in ["asc", "desc"]:
+        response = APIClient().post(
+            "/api/v1.0/documents/search/",
+            {
+                "q": "*",
+                "order_by": "relevance",
+                "order_direction": direction,
+                "visited": [doc["id"] for doc in documents],
+            },
+            format="json",
+            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+        )
+
+        assert response.status_code == 200
+        data = response.json()
+        assert len(data) == 4
+
+        # Check that results are sorted by score as expected
+        compare = operator.le if direction == "asc" else operator.ge
+        for i in range(len(data) - 1):
+            assert compare(data[i]["_score"], data[i + 1]["_score"])
+
+
+@responses.activate
+def test_api_documents_search_ordering_by_unknown_field(settings):
+    """Trying to sort by an unknown field should return a 400 error"""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    # Setup: Initialize the service and documents only once
+    service = factories.ServiceFactory()
+    documents = factories.DocumentSchemaFactory.build_batch(
+        2, reach=random.choice(["public", "authenticated"])
+    )
+    prepare_index(service.index_name, documents)
+
+    # Define the parameters manually
+    directions = ["asc", "desc"]
+
+    # Perform the parameterized tests
+    for direction in directions:
+        response = APIClient().post(
+            "/api/v1.0/documents/search/",
+            {
+                "q": "*",
+                "order_by": "unknown",
+                "order_direction": direction,
+                "visited": [doc["id"] for doc in documents],
+            },
+            format="json",
+            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+        )
+
+        assert response.status_code == 400
+        assert response.json() == [
+            {
+                "loc": ["order_by"],
+                "msg": (
+                    "Input should be 'relevance', 'title', 'created_at', "
+                    "'updated_at', 'size' or 'reach'"
+                ),
+                "type": "literal_error",
+            }
+        ]
+
+
+@responses.activate
+def test_api_documents_search_ordering_by_unknown_direction(settings):
+    """Trying to sort with an unknown direction should return a 400 error"""
+    setup_oicd_resource_server(responses, settings, sub="user_sub")
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+    service = factories.ServiceFactory()
+    documents = factories.DocumentSchemaFactory.build_batch(
+        2, reach=random.choice(["public", "authenticated"])
+    )
+    prepare_index(service.index_name, documents)
+
+    for field in enums.ORDER_BY_OPTIONS:
+        response = APIClient().post(
+            "/api/v1.0/documents/search/",
+            {
+                "q": "*",
+                "order_by": field,
+                "order_direction": "unknown",
+                "visited": [doc["id"] for doc in documents],
+            },
+            format="json",
+            HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
+        )
+
+        assert response.status_code == 400
+        assert response.json() == [
+            {
+                "loc": ["order_direction"],
+                "msg": "Input should be 'asc' or 'desc'",
+                "type": "literal_error",
+            }
+        ]
+
+
 @responses.activate
 def test_api_documents_search_filtering_by_reach(settings):
     """It should be possible to filter results by their reach"""
@@ -561,7 +733,6 @@ def test_api_documents_search_with_nb_results(settings):
             "q": "*",
             "nb_results": nb_results,
             "visited": [doc["id"] for doc in documents],
-            "rescore": False,
         },
         format="json",
         HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
@@ -584,7 +755,7 @@ def test_api_documents_search_with_nb_results(settings):
     )
     assert response.status_code == 200
     data = response.json()
-    assert {r["_id"] for r in data} == set(ids[0:nb_results])
+    assert [r["_id"] for r in data] == ids[0:nb_results]
 
     nb_results = 10
     response = APIClient().post(
@@ -600,7 +771,7 @@ def test_api_documents_search_with_nb_results(settings):
     assert response.status_code == 200
     data = response.json()
     # nb_results > total number of documents => returns all documents
-    assert {r["_id"] for r in data} == set(ids[0:9])
+    assert [r["_id"] for r in data] == ids[0:9]
 
 
 @responses.activate
@@ -669,13 +840,12 @@ def test_api_documents_search_nb_results_with_filtering(settings):
             "reach": "public",
             "nb_results": nb_results,
             "visited": public_ids,
-            "rescore_enable": False,
         },
         format="json",
         HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}",
     )
     assert response.status_code == 200
-    assert {r["_id"] for r in response.json()} == set(public_ids[0:nb_results])
+    assert [r["_id"] for r in response.json()] == public_ids[0:nb_results]
 
 
 @responses.activate

src/backend/core/tests/test_search.py

@@ -2,8 +2,8 @@
 Test suite for opensearch search service
 """
 
-import datetime
 import logging
+import operator
 from json import dumps as json_dumps
 
 import pytest
@@ -32,6 +32,8 @@ def search_params(service):
     """Build opensearch.search() parameters for tests using the service index name"""
     return {
         "nb_results": 20,
+        "order_by": "relevance",
+        "order_direction": "desc",
         "search_indices": {service.index_name},
         "reach": None,
         "user_sub": "user_sub",
@@ -405,6 +407,41 @@ def test_match_all(settings, caplog):
     assert len(result["hits"]["hits"]) == 3
 
 
+@responses.activate
+def test_search_ordering_by_relevance(settings, caplog):
+    """Test the hybrid supports ordering by relevance asc and desc"""
+    enable_hybrid_search(settings)
+    responses.add(
+        responses.POST,
+        settings.EMBEDDING_API_PATH,
+        json=albert_embedding_response.response,
+        status=200,
+    )
+
+    documents = bulk_create_documents(
+        [
+            {"title": "wolf", "content": "wolves live in packs and hunt together"},
+            {"title": "dog", "content": "dogs are loyal domestic animals"},
+            {"title": "cat", "content": "cats are curious and independent pets"},
+        ]
+    )
+    q = "canine pet"
+    service = factories.ServiceFactory(name=SERVICE_NAME)
+    prepare_index(service.index_name, documents)
+
+    for direction in ["asc", "desc"]:
+        with caplog.at_level(logging.INFO):
+            result = search(
+                q=q, **{**search_params(service), "order_direction": direction}
+            )
+
+        # Check that results are sorted by score as expected
+        hits = result["hits"]["hits"]
+        compare = operator.le if direction == "asc" else operator.ge
+        for i in range(len(hits) - 1):
+            assert compare(hits[i]["_score"], hits[i + 1]["_score"])
+
+
 @responses.activate
 def test_hybrid_search_number_of_matches(settings):
     """
@@ -675,43 +712,3 @@ def test_search_filtering_by_query_path_and_tag():
 
     assert result["hits"]["total"]["value"] == len(documents_to_search)
     assert returned_ids == expected_ids
-
-
-def test_search_with_rescore(settings):
-    """Test rescore feature"""
-    service = factories.ServiceFactory(name=SERVICE_NAME)
-
-    today = datetime.datetime.today()
-    forty_days_ago = today - datetime.timedelta(days=40)
-    documents = [
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="my first document",
-            created_at=today,
-            updated_at=today,
-        ),
-        factories.DocumentSchemaFactory.build(
-            users=["user_sub"],
-            title="another document",
-            created_at=forty_days_ago,
-            updated_at=forty_days_ago,
-        ),
-    ]
-    prepare_index(service.index_name, documents)
-
-    # set a cray big RESCORE_UPDATED_AT_WEIGHT to demonstrate the effect of boosting on rescores
-    settings.RESCORE_UPDATED_AT_WEIGHT = 200
-
-    results = search(
-        q="another document",
-        **{
-            **search_params(service),
-            "rescore": True,
-        },
-    )
-
-    hits = results["hits"]["hits"]
-    # the first document is ranked first because it more recent
-    # even though the second one matches the query better
-    assert hits[0]["_source"]["title.en"] == "my first document"
-    assert hits[1]["_source"]["title.en"] == "another document"

src/backend/core/views.py

@@ -332,6 +332,12 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
         path : str, optional
             Filter results based on the 'path' field. Only documents whose path
             starts with the provided value will be returned.
+        order_by : str, optional
+            Order results by 'relevance', 'created_at', 'updated_at', or 'size'.
+            Defaults to 'relevance' if not specified.
+        order_direction : str, optional
+            Order direction, 'asc' for ascending or 'desc' for descending.
+            Defaults to 'desc'.
         nb_results : int, optional
             The number of results to return.
             Defaults to 50 if not specified.
@@ -375,6 +381,8 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
         result = search(
             q=params.q,
             nb_results=params.nb_results,
+            order_by=params.order_by,
+            order_direction=params.order_direction,
             search_indices=search_indices,
             reach=params.reach,
             visited=params.visited,
@@ -387,7 +395,6 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
             else SearchTypeEnum.HYBRID
             if check_hybrid_search_enabled()
             else SearchTypeEnum.FULL_TEXT,
-            rescore=params.rescore,
         )["hits"]["hits"]
         logger.info("found %d results", len(result))
         logger.debug("results %s", result)

src/backend/evaluation/management/commands/evaluate_search_engine.py

@@ -38,6 +38,8 @@ class Command(BaseCommand):
     index_name = "evaluation-index"
     search_params = {
         "nb_results": 20,
+        "order_by": "relevance",
+        "order_direction": "desc",
         "search_indices": {index_name},
         "reach": None,
         "user_sub": "user_sub",

src/backend/find/settings.py

@@ -318,19 +318,6 @@ class Base(Configuration):
     EMBEDDING_DIMENSION = values.IntegerValue(
         default=1024, environ_name="EMBEDDING_DIMENSION", environ_prefix=None
     )
-    # rescore
-    RESCORE_UPDATED_AT_WEIGHT = values.FloatValue(
-        default=0.2, environ_name="RESCORE_UPDATED_AT_WEIGHT", environ_prefix=None
-    )
-    RESCORE_UPDATED_AT_OFFSET = values.Value(
-        default="2d", environ_name="RESCORE_UPDATED_AT_OFFSET", environ_prefix=None
-    )
-    RESCORE_UPDATED_AT_SCALE = values.Value(
-        default="6d", environ_name="RESCORE_UPDATED_AT_SCALE", environ_prefix=None
-    )
-    RESCORE_UPDATED_AT_DECAY = values.IntegerValue(
-        default=0.5, environ_name="RESCORE_UPDATED_AT_SCALE", environ_prefix=None
-    )
 
     # CORS
     CORS_ALLOW_CREDENTIALS = True