From 2c090551c00037ca112a56718dbf6b21e1028171 Mon Sep 17 00:00:00 2001 From: Charles Englebert <101663613+mascarpon3@users.noreply.github.com> Date: Mon, 9 Feb 2026 16:45:18 +0100 Subject: [PATCH] Filter on path (#53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨(backend) add path filter allow search filters on path for Docs compatibility * ✅(backend) test I improve existing and add new tests --- CHANGELOG.md | 3 +- src/backend/core/schemas.py | 1 + src/backend/core/services/indexing.py | 25 ++ src/backend/core/services/search.py | 17 +- .../core/tests/test_api_documents_search.py | 99 +++++-- src/backend/core/tests/test_search.py | 266 +++++++++++++----- src/backend/core/views.py | 34 +-- 7 files changed, 321 insertions(+), 124 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e2c27f..6ac8e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,8 @@ and this project adheres to issues if the opensearch database is shared between apps. - ✨(backend) add tags - ✨(backend) adapt to conversation RAG -- ✨(backend) add deletion endpoint +- ✨(backend) add deletion endpoint +- ✨(backend) add path filter ## Changed diff --git a/src/backend/core/schemas.py b/src/backend/core/schemas.py index 8318e94..b7db928 100644 --- a/src/backend/core/schemas.py +++ b/src/backend/core/schemas.py @@ -114,6 +114,7 @@ class SearchQueryParametersSchema(BaseModel): visited: StringListParameter = Field(default_factory=list) reach: Optional[enums.ReachEnum] = None tags: StringListParameter = Field(default_factory=list) + path: Optional[str] = None order_by: Optional[Literal[enums.ORDER_BY_OPTIONS]] = Field(default=enums.RELEVANCE) order_direction: Optional[Literal["asc", "desc"]] = Field(default="desc") nb_results: Optional[conint(ge=1, le=300)] = Field(default=50) diff --git a/src/backend/core/services/indexing.py b/src/backend/core/services/indexing.py index f16cbbf..13ec59c 100644 --- a/src/backend/core/services/indexing.py +++ b/src/backend/core/services/indexing.py @@ -3,6 +3,7 @@ import logging from django.conf import settings +from django.core.exceptions import SuspiciousOperation from langchain_text_splitters import RecursiveCharacterTextSplitter from opensearchpy.exceptions import NotFoundError @@ -14,6 +15,7 @@ from core.services.opensearch_configuration import ( MAPPINGS, ) +from ..models import Service, get_opensearch_index_name from .embedding import embed_text from .opensearch import check_hybrid_search_enabled, opensearch_client @@ -124,3 +126,26 @@ def detect_language_code(text): return settings.UNDETERMINED_LANGUAGE_CODE return detected_code + + +def get_opensearch_indices(audience, services): + """ + Get OpenSearch indices for the given audience and services. + """ + try: + user_service = Service.objects.get(client_id=audience, is_active=True) + except Service.DoesNotExist as e: + logger.warning("Login failed: No service %s found", audience) + raise SuspiciousOperation("Service is not available") from e + + # Find allowed sub-services for this service + allowed_services = set(user_service.services.values_list("name", flat=True)) + allowed_services.add(user_service.name) + + if services: + available_service = set(services).intersection(allowed_services) + + if len(available_service) < len(services): + raise SuspiciousOperation("Some requested services are not available") + + return [get_opensearch_index_name(service) for service in allowed_services] diff --git a/src/backend/core/services/search.py b/src/backend/core/services/search.py index efe0dac..41a4865 100644 --- a/src/backend/core/services/search.py +++ b/src/backend/core/services/search.py @@ -24,6 +24,7 @@ def search( # noqa : PLR0913 user_sub, groups, tags, + path=None, ): """Perform an OpenSearch search""" query = get_query( @@ -34,6 +35,7 @@ def search( # noqa : PLR0913 user_sub=user_sub, groups=groups, tags=tags, + path=path, ) return opensearch_client().search( # pylint: disable=unexpected-keyword-arg index=",".join(search_indices), @@ -49,7 +51,6 @@ def search( # noqa : PLR0913 order_direction=order_direction, ), "size": nb_results, - # Compute query "query": query, }, params=get_params(query_keys=query.keys()), @@ -61,10 +62,10 @@ def search( # noqa : PLR0913 # pylint: disable=too-many-arguments, too-many-positional-arguments def get_query( # noqa : PLR0913 - q, nb_results, reach, visited, user_sub, groups, tags + q, nb_results, reach, visited, user_sub, groups, tags, path=None ): """Build OpenSearch query body based on parameters""" - filter_ = get_filter(reach, visited, user_sub, groups, tags) + filter_ = get_filter(reach, visited, user_sub, groups, tags, path) if q == "*": logger.info("Performing match_all query") @@ -155,7 +156,10 @@ def get_full_text_query(q, filter_): } -def get_filter(reach, visited, user_sub, groups, tags): +# pylint: disable=too-many-arguments, too-many-positional-arguments +def get_filter( # noqa : PLR0913 + reach, visited, user_sub, groups, tags, path=None +): """Build OpenSearch filter""" filters = [ {"term": {"is_active": True}}, # filter out inactive documents @@ -192,6 +196,11 @@ def get_filter(reach, visited, user_sub, groups, tags): # logical or: if tags are provided the matching documents should have at least one of them filters.append({"terms": {"tags": tags}}) + # Optional path filter + if path: + # filter documents that start with the provided path + filters.append({"prefix": {"path": path}}) + return filters diff --git a/src/backend/core/tests/test_api_documents_search.py b/src/backend/core/tests/test_api_documents_search.py index d597291..2a9aa98 100644 --- a/src/backend/core/tests/test_api_documents_search.py +++ b/src/backend/core/tests/test_api_documents_search.py @@ -100,8 +100,6 @@ def test_api_documents_search_query_unknown_user(settings): introspect=lambda request, user_info: (404, {}, ""), ) - token = build_authorization_bearer() - service = factories.ServiceFactory() prepare_index(service.index_name, []) @@ -109,7 +107,7 @@ def test_api_documents_search_query_unknown_user(settings): "/api/v1.0/documents/search/", {"q": "a quick fox"}, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 400 @@ -179,7 +177,6 @@ def test_api_documents_search_reached_docs_invalid_parameters(settings): def test_api_documents_search_match_all(settings): """Searching a document with q='*' should match all docs""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -197,7 +194,7 @@ def test_api_documents_search_match_all(settings): "/api/v1.0/documents/search/", {"q": "*", "visited": [doc["id"] for doc in documents]}, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 @@ -284,7 +281,6 @@ def test_api_documents_full_text_search(settings): Searching a document by its content should work as expected. """ setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() service = factories.ServiceFactory() documents = bulk_create_documents( @@ -300,7 +296,7 @@ def test_api_documents_full_text_search(settings): "/api/v1.0/documents/search/", {"q": "a quick fox", "visited": [doc["id"] for doc in documents]}, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 @@ -358,7 +354,6 @@ def test_api_documents_full_text_search(settings): def test_api_documents_hybrid_search(settings): """Searching a document by its content should work as expected""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() # hybrid search is enabled by default enable_hybrid_search(settings) responses.add( @@ -382,7 +377,7 @@ def test_api_documents_hybrid_search(settings): "/api/v1.0/documents/search/", {"q": "a quick fox", "visited": [doc["id"] for doc in documents]}, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 @@ -469,7 +464,6 @@ def test_api_documents_hybrid_search(settings): def test_api_documents_search_ordering_by_fields(settings): """It should be possible to order by several fields""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -503,7 +497,7 @@ def test_api_documents_search_ordering_by_fields(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 @@ -520,7 +514,6 @@ def test_api_documents_search_ordering_by_fields(settings): def test_api_documents_search_ordering_by_relevance(settings): """It should be possible to order by relevance (score)""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -543,7 +536,7 @@ def test_api_documents_search_ordering_by_relevance(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 @@ -560,7 +553,6 @@ def test_api_documents_search_ordering_by_relevance(settings): def test_api_documents_search_ordering_by_unknown_field(settings): """Trying to sort by an unknown field should return a 400 error""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -588,7 +580,7 @@ def test_api_documents_search_ordering_by_unknown_field(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 400 @@ -608,7 +600,6 @@ def test_api_documents_search_ordering_by_unknown_field(settings): def test_api_documents_search_ordering_by_unknown_direction(settings): """Trying to sort with an unknown direction should return a 400 error""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -631,7 +622,7 @@ def test_api_documents_search_ordering_by_unknown_direction(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 400 @@ -648,7 +639,6 @@ def test_api_documents_search_ordering_by_unknown_direction(settings): def test_api_documents_search_filtering_by_reach(settings): """It should be possible to filter results by their reach""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -670,7 +660,7 @@ def test_api_documents_search_filtering_by_reach(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 @@ -684,7 +674,6 @@ def test_api_documents_search_filtering_by_reach(settings): def test_api_documents_search_with_nb_results(settings): """nb_size should correctly return results of given size""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -707,7 +696,7 @@ def test_api_documents_search_with_nb_results(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 @@ -723,7 +712,7 @@ def test_api_documents_search_with_nb_results(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 data = response.json() @@ -738,7 +727,7 @@ def test_api_documents_search_with_nb_results(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 data = response.json() @@ -750,7 +739,6 @@ def test_api_documents_search_with_nb_results(settings): def test_api_documents_search_nb_results_invalid_parameters(settings): """Invalid nb_results parameters should result in a 400 error""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -779,7 +767,7 @@ def test_api_documents_search_nb_results_invalid_parameters(settings): "/api/v1.0/documents/search/", {"q": "*", "nb_results": nb_results}, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 400 @@ -791,7 +779,6 @@ def test_api_documents_search_nb_results_invalid_parameters(settings): def test_api_documents_search_nb_results_with_filtering(settings): """nb_results should work correctly when combined with filtering by reach""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -816,7 +803,7 @@ def test_api_documents_search_nb_results_with_filtering(settings): "visited": public_ids, }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 assert [r["_id"] for r in response.json()] == public_ids[0:nb_results] @@ -826,7 +813,6 @@ def test_api_documents_search_nb_results_with_filtering(settings): def test_api_documents_search_filtering_by_tags(settings): """Test filtering documents by a single tag via API""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -865,7 +851,7 @@ def test_api_documents_search_filtering_by_tags(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 @@ -878,7 +864,6 @@ def test_api_documents_search_filtering_by_tags(settings): def test_api_documents_search_without_tags_filter(settings): """Test that search works normally when no tags filter is provided""" setup_oicd_resource_server(responses, settings, sub="user_sub") - token = build_authorization_bearer() responses.add( responses.POST, settings.EMBEDDING_API_PATH, @@ -912,8 +897,60 @@ def test_api_documents_search_without_tags_filter(settings): "visited": [doc["id"] for doc in documents], }, format="json", - HTTP_AUTHORIZATION=f"Bearer {token}", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", ) assert response.status_code == 200 assert len(response.json()) == 2 + + +@responses.activate +def test_api_documents_search_filtering_by_path(settings): + """Test filtering documents by path prefix via API""" + setup_oicd_resource_server(responses, settings, sub="user_sub") + responses.add( + responses.POST, + settings.EMBEDDING_API_PATH, + json=albert_embedding_response.response, + status=200, + ) + service = factories.ServiceFactory() + + documents = bulk_create_documents( + [ + { + "title": "Document with tags", + "content": "Tagged document", + "path": "/path/to/doc1", + }, + { + "title": "Document without tags", + "content": "Untagged document", + "path": "/path/to/doc2", + }, + { + "title": "Document without tags", + "content": "Untagged document", + "path": "other/path/to/doc3", + }, + ] + ) + + prepare_index(service.index_name, documents) + + path_filter = "/path/to/" + response = APIClient().post( + "/api/v1.0/documents/search/", + { + "q": "*", + "path": path_filter, + "visited": [doc["id"] for doc in documents], + }, + format="json", + HTTP_AUTHORIZATION=f"Bearer {build_authorization_bearer()}", + ) + + assert response.status_code == 200 + assert len(response.json()) == 2 + for hit in response.json(): + assert hit["_source"]["path"].startswith(path_filter) diff --git a/src/backend/core/tests/test_search.py b/src/backend/core/tests/test_search.py index 9a8d77b..b6c9b8a 100644 --- a/src/backend/core/tests/test_search.py +++ b/src/backend/core/tests/test_search.py @@ -367,75 +367,73 @@ def test_search_filtering_by_single_tag(): """Test filtering documents by a single tag""" service = factories.ServiceFactory(name=SERVICE_NAME) - documents = bulk_create_documents( - [ - { - "title": "Document with python tag", - "content": "This is about Python programming", - "tags": ["python", "programming"], - }, - { - "title": "Document with javascript tag", - "content": "This is about JavaScript", - "tags": ["javascript", "programming"], - }, - { - "title": "Document with no tags", - "content": "This has no tags", - "tags": [], - }, - ] - ) + documents_to_search = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], tags=["tag-to-search"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], tags=["tag-to-search", "tag-to-filter"] + ), + ] + documents_to_filter = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], tags=["tag-to-filter"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + ), + ] + expected_ids = {str(doc["id"]) for doc in documents_to_search} - prepare_index(service.index_name, documents) + prepare_index(service.index_name, documents_to_search + documents_to_filter) - # Search for documents with python tag - result = search(q="*", **{**search_params(service), "tags": ["python"]}) + # Search for documents with tag-to-search tag + result = search(q="*", **{**search_params(service), "tags": ["tag-to-search"]}) + returned_ids = {hit["_id"] for hit in result["hits"]["hits"]} - assert result["hits"]["total"]["value"] == 1 - assert result["hits"]["hits"][0]["_id"] == str(documents[0]["id"]) + assert result["hits"]["total"]["value"] == len(documents_to_search) + assert returned_ids == expected_ids def test_search_filtering_by_multiple_tags(): """Test filtering documents by multiple tags (OR logic)""" service = factories.ServiceFactory(name=SERVICE_NAME) - documents = bulk_create_documents( - [ - { - "title": "Document with python tag", - "content": "This is about Python programming", - "tags": ["python", "backend"], - }, - { - "title": "Document with javascript tag", - "content": "This is about JavaScript", - "tags": ["javascript", "frontend"], - }, - { - "title": "Document with java tag", - "content": "This is about Java", - "tags": ["java", "backend"], - }, - { - "title": "Document with no tags", - "content": "This has no tags", - "tags": [], - }, - ] - ) + documents_to_search = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], tags=["tag-to-search-1"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], tags=["tag-to-search-1", "tag-to-filter"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], tags=["tag-to-search-2", "tag-to-filter"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], tags=["tag-to-search-1", "tag-to-search-2"] + ), + ] + documents_to_filter = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], tags=["tag-to-filter"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + ), + ] + expected_ids = {str(doc["id"]) for doc in documents_to_search} - prepare_index(service.index_name, documents) + prepare_index(service.index_name, documents_to_search + documents_to_filter) - # Search for documents with python OR javascript tags + # Search for documents with tag-to-search-1 OR tag-to-search-2 tags result = search( - q="*", **{**search_params(service), "tags": ["python", "javascript"]} + q="*", + **{**search_params(service), "tags": ["tag-to-search-1", "tag-to-search-2"]}, ) - - assert result["hits"]["total"]["value"] == 2 returned_ids = {hit["_id"] for hit in result["hits"]["hits"]} - assert str(documents[0]["id"]) in returned_ids - assert str(documents[1]["id"]) in returned_ids + + assert result["hits"]["total"]["value"] == len(documents_to_search) + assert returned_ids == expected_ids def test_search_no_tags_filter_returns_all(): @@ -444,22 +442,164 @@ def test_search_no_tags_filter_returns_all(): documents = bulk_create_documents( [ - { - "title": "Document with tags", - "content": "Tagged document", - "tags": ["python"], - }, { "title": "Document without tags", "content": "Untagged document", - "tags": [], + }, + { + "title": "Document with tags", + "content": "Tagged document", + "tags": ["tag-to-search"], }, ] ) - prepare_index(service.index_name, documents) # Search without tags filter result = search(q="*", **search_params(service)) - assert result["hits"]["total"]["value"] == 2 + assert result["hits"]["total"]["value"] == len(documents) + + +def test_search_filtering_by_tag_and_query(): + """Test filtering documents by both tag and query text""" + service = factories.ServiceFactory(name=SERVICE_NAME) + + documents_to_search = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], title="title to search", tags=["tag-to-search"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to search 1", + tags=["tag-to-search", "tag-to-filter"], + ), + ] + documents_to_filter = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], title="title to filter", tags=["tag-to-filter"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], title="title to filter 1", tags=["tag-to-filter"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to search 2", + ), + ] + expected_ids = {str(doc["id"]) for doc in documents_to_search} + + prepare_index(service.index_name, documents_to_search + documents_to_filter) + + # Search with both query and tag filter + result = search(q="search", **{**search_params(service), "tags": ["tag-to-search"]}) + returned_ids = {hit["_id"] for hit in result["hits"]["hits"]} + + assert result["hits"]["total"]["value"] == len(documents_to_search) + assert returned_ids == expected_ids + + +def test_search_filtering_by_path(): + """Test filtering documents by path prefix""" + service = factories.ServiceFactory(name=SERVICE_NAME) + + documents_to_search = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], path="path/to/search/doc-1" + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], path="path/to/search/doc-2" + ), + ] + documents_to_filter = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], path="path/to/filter/doc-3" + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + ), + ] + expected_ids = {str(doc["id"]) for doc in documents_to_search} + + prepare_index(service.index_name, documents_to_search + documents_to_filter) + + path_filter = "path/to/search" + result = search(q="*", **{**search_params(service), "path": path_filter}) + returned_ids = {hit["_id"] for hit in result["hits"]["hits"]} + + assert result["hits"]["total"]["value"] == len(documents_to_search) + assert returned_ids == expected_ids + + +def test_search_filtering_by_query_path_and_tag(): + """Test filtering documents by query text, path prefix and tag combined""" + service = factories.ServiceFactory(name=SERVICE_NAME) + + documents_to_search = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to search 0", + path="path/to/search-0", + tags=["tag-to-search"], + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to search 1", + path="path/to/search/doc1", + tags=["tag-to-search", "tag-to-filter"], + ), + ] + documents_to_filter = [ + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to filter", + path="path/to/search/doc-3", + tags=["tag-to-search"], + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to search 4", + path="path/to/filter/doc-4", + tags=["tag-to-search"], + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to search 4", + path="path/to/search/doc-4", + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], title="title to search 5", tags=["tag-to-search"] + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to search 6", + path="path/to/search/doc-6", + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="title to search 7", + ), + factories.DocumentSchemaFactory.build( + users=["user_sub"], + title="", + path="path/to/search/doc-8", + tags=["tag-to-search"], + ), + ] + expected_ids = {str(doc["id"]) for doc in documents_to_search} + + prepare_index(service.index_name, documents_to_search + documents_to_filter) + + # Search with query, path and tag filters combined + result = search( + q="search", + **{ + **search_params(service), + "path": "path/to/search", + "tags": ["tag-to-search"], + }, + ) + returned_ids = {hit["_id"] for hit in result["hits"]["hits"]} + + assert result["hits"]["total"]["value"] == len(documents_to_search) + assert returned_ids == expected_ids diff --git a/src/backend/core/views.py b/src/backend/core/views.py index 4eeed9a..c39e9bc 100644 --- a/src/backend/core/views.py +++ b/src/backend/core/views.py @@ -12,9 +12,12 @@ from rest_framework.response import Response from . import schemas from .authentication import ServiceTokenAuthentication -from .models import Service, get_opensearch_index_name from .permissions import IsAuthAuthenticated -from .services.indexing import ensure_index_exists, prepare_document_for_indexing +from .services.indexing import ( + ensure_index_exists, + get_opensearch_indices, + prepare_document_for_indexing, +) from .services.opensearch import opensearch_client from .services.search import search from .utils import get_language_value @@ -325,6 +328,9 @@ class SearchDocumentView(ResourceServerMixin, views.APIView): tags : List[str], optional Filter results based on the 'tags' field. Documents matching any of the provided tags will be returned. + path : str, optional + Filter results based on the 'path' field. Only documents whose path + starts with the provided value will be returned. order_by : str, optional Order results by 'relevance', 'created_at', 'updated_at', or 'size'. Defaults to 'relevance' if not specified. @@ -375,31 +381,9 @@ class SearchDocumentView(ResourceServerMixin, views.APIView): user_sub=user_sub, groups=groups, tags=params.tags, + path=params.path, )["hits"]["hits"] logger.info("found %d results", len(result)) logger.debug("results %s", result) return Response(result, status=status.HTTP_200_OK) - - -def get_opensearch_indices(audience, services): - """ - Get OpenSearch indices for the given audience and services. - """ - try: - user_service = Service.objects.get(client_id=audience, is_active=True) - except Service.DoesNotExist as e: - logger.warning("Login failed: No service %s found", audience) - raise SuspiciousOperation("Service is not available") from e - - # Find allowed sub-services for this service - allowed_services = set(user_service.services.values_list("name", flat=True)) - allowed_services.add(user_service.name) - - if services: - available_service = set(services).intersection(allowed_services) - - if len(available_service) < len(services): - raise SuspiciousOperation("Some requested services are not available") - - return [get_opensearch_index_name(service) for service in allowed_services]