Adapt to conversation (#30)

* ✨(backend) enhance document indexing with error handling and changelog

I enhance document indexing with error handling and changelog

* ✨(backend) adapt to conversation RAG

the document rag tool of conversation expect a content
and a dedicated service with token access is needed.
I am also adding loggers.

* 🚨(backend) fix tests

demo tests are broken. Here is the fix.

CHANGELOG.md

@@ -20,14 +20,13 @@ and this project adheres to
 - ✨(backend) allow indexation of documents with either empty content or title.
 - ✨(api) new fulltext 'search/' view with OIDC resource server authentication
 - ✨(backend) limit access to documents : public & authenticated with a
-              linkreach & owned ones
-- ✨(backend) limit search to the calling app (audience) and a configured
               list of services
 - 🔧(compose) rename docker network 'lasuite-net' as 'lasuite-network'
 - ✨(backend) add demo service for Drive.
-- ✨(backend) Add OPENSEARCH_INDEX_PREFIX setting to prevent naming overlaping
+- ✨(backend) add OPENSEARCH_INDEX_PREFIX setting to prevent naming overlaping
               issues if the opensearch database is shared between apps.
 - ✨(backend) add tags
+- ✨(backend) adapt to conversation RAG
 
 ## Fixed
 

src/backend/core/enums.py

@@ -23,6 +23,7 @@ REACH = "reach"
 SIZE = "size"
 TAGS = "tags"
 TITLE = "title"
+CONTENT = "content"
 UPDATED_AT = "updated_at"
 USERS = "users"
 GROUPS = "groups"
@@ -32,6 +33,7 @@ RELEVANCE = "relevance"
 ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, REACH)
 SOURCE_FIELDS = (
     TITLE,
+    CONTENT,
     SIZE,
     DEPTH,
     PATH,

src/backend/core/services/search.py

@@ -54,7 +54,7 @@ def search(  # noqa : PLR0913
         },
         params=get_params(query_keys=query.keys()),
         # disable=unexpected-keyword-arg because
-        # ignore_unavailable is not in the the method declaration
+        # ignore_unavailable is not in the method declaration
         ignore_unavailable=True,
     )
 

src/backend/core/tests/test_api_documents_search.py

@@ -237,6 +237,7 @@ def test_api_documents_full_text_search_query_title(settings):
     assert list(fox_response.keys()) == ["_index", "_id", "_score", "_source", "fields"]
     assert fox_response["_id"] == str(documents[0]["id"])
     assert fox_response["_source"] == {
+        "content.en": "the wolf",
         "depth": 1,
         "numchild": 0,
         "path": fox_document["path"],
@@ -260,6 +261,7 @@ def test_api_documents_full_text_search_query_title(settings):
     ]
     assert other_fox_response["_id"] == str(other_fox_document["id"])
     assert other_fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": other_fox_document["path"],
@@ -310,6 +312,7 @@ def test_api_documents_full_text_search(settings):
     assert fox_response["_id"] == str(fox_document["id"])
     assert fox_response["_score"] > 0
     assert fox_response["_source"] == {
+        "content.en": "the wolf",
         "depth": 1,
         "numchild": 0,
         "path": fox_document["path"],
@@ -334,6 +337,7 @@ def test_api_documents_full_text_search(settings):
     assert other_fox_response["_id"] == str(other_fox_document["id"])
     assert other_fox_response["_score"] > 0
     assert other_fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": other_fox_document["path"],
@@ -392,6 +396,7 @@ def test_api_documents_hybrid_search(settings):
     assert fox_response["_id"] == str(fox_document["id"])
     assert fox_response["_score"] > 0
     assert fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": fox_document["path"],
@@ -416,6 +421,7 @@ def test_api_documents_hybrid_search(settings):
     assert other_fox_response["_id"] == str(other_fox_document["id"])
     assert other_fox_response["_score"] > 0
     assert other_fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": other_fox_document["path"],
@@ -442,6 +448,7 @@ def test_api_documents_hybrid_search(settings):
     ]
     assert no_fox_response["_id"] == str(no_fox_document["id"])
     assert no_fox_response["_source"] == {
+        "content.en": fox_document["content"],
         "depth": 1,
         "numchild": 0,
         "path": no_fox_document["path"],

src/backend/core/views.py

@@ -17,6 +17,7 @@ from .permissions import IsAuthAuthenticated
 from .services.indexing import ensure_index_exists, prepare_document_for_indexing
 from .services.opensearch import opensearch_client
 from .services.search import search
+from .utils import get_language_value
 
 logger = logging.getLogger(__name__)
 
@@ -106,6 +107,11 @@ class IndexDocumentView(views.APIView):
             schemas.DocumentSchema(**request.data).model_dump()
         )
         _id = document_dict.pop("id")
+        logger.info(
+            "Indexing single document %s on index %s",
+            get_language_value(document_dict, "title"),
+            index_name,
+        )
 
         ensure_index_exists(index_name)
         opensearch_client_.index(
@@ -149,6 +155,11 @@ class IndexDocumentView(views.APIView):
                 has_errors = True
             else:
                 document_dict = prepare_document_for_indexing(document.model_dump())
+                logger.info(
+                    "Indexing document %s on index %s",
+                    get_language_value(document_dict, "title"),
+                    index_name,
+                )
                 _id = document_dict.pop("id")
                 actions.append({"index": {"_id": _id}})
                 actions.append(document_dict)
@@ -259,7 +270,8 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
         except SuspiciousOperation as e:
             return Response({"detail": str(e)}, status=status.HTTP_400_BAD_REQUEST)
 
-        response = search(
+        logger.info("Search '%s' on indices %s", params.q, search_indices)
+        result = search(
             q=params.q,
             nb_results=params.nb_results,
             order_by=params.order_by,
@@ -270,6 +282,8 @@ class SearchDocumentView(ResourceServerMixin, views.APIView):
             user_sub=user_sub,
             groups=groups,
             tags=params.tags,
-        )
+        )["hits"]["hits"]
+        logger.info("found %d results", len(result))
+        logger.debug("results %s", result)
 
-        return Response(response["hits"]["hits"], status=status.HTTP_200_OK)
+        return Response(result, status=status.HTTP_200_OK)

src/backend/demo/defaults.py

@@ -13,4 +13,9 @@ DEV_SERVICES = (
         "client_id": "drive",
         "token": "find-api-key-for-driv-with-exactly-50-chars-length",
     },
+    {
+        "name": "conversations",
+        "client_id": "conversations",
+        "token": "find-api-key-for-conv-with-exactly-50-chars-length",
+    },
 )

src/backend/demo/tests/test_commands_create_demo.py

@@ -26,7 +26,7 @@ def test_commands_create_demo():
     """The create_demo management command should create objects as expected."""
     call_command("create_demo")
 
-    assert models.Service.objects.exclude(name="docs").count() == 3
+    assert models.Service.objects.exclude(name="docs").count() == 4
     assert opensearch_client().count()["count"] == 4
 
     docs = models.Service.objects.get(name="docs")

src/backend/find/settings.py

@@ -130,6 +130,7 @@ class Base(Configuration):
             ("en", _("English")),
             ("de", _("German")),
             ("nl", _("Dutch")),
+            ("und", None),
         )
     )
     SUPPORTED_LANGUAGE_CODES = tuple(