From 624da64dcbc2693ffc71c4e6c208eee2cd589c41 Mon Sep 17 00:00:00 2001 From: Fabre Florian Date: Fri, 7 Nov 2025 16:17:39 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8(backend)=20prefix=20for=20service=20i?= =?UTF-8?q?ndex=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Service opensearch index is now defined by the property 'index_name' and prefixed by the new setting OPENSEARCH_INDEX_PREFIX (default: 'find'). Fix parallel execution of tests. Signed-off-by: Fabre Florian --- CHANGELOG.md | 3 + .../commands/reindex_with_embedding.py | 3 +- src/backend/core/models.py | 12 ++++ .../commands/test_reindex_with_embedding.py | 42 +++++++------ src/backend/core/tests/conftest.py | 29 +++++++++ .../tests/test_api_documents_index_bulk.py | 27 ++++---- .../tests/test_api_documents_index_single.py | 36 +++++------ .../core/tests/test_api_documents_search.py | 63 ++++++++++--------- ...est_api_documents_search_access_control.py | 48 +++++++------- .../core/tests/test_models_services.py | 3 +- src/backend/core/tests/test_opensearch.py | 19 ++++-- src/backend/core/tests/utils.py | 10 +-- src/backend/core/views.py | 6 +- src/backend/find/settings.py | 3 + 14 files changed, 177 insertions(+), 127 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f30b54a..d37b01b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,3 +22,6 @@ and this project adheres to list of services - 🔧(compose) rename docker network 'lasuite-net' as 'lasuite-network' - ✨(backend) add demo service for Drive. +- 🐛(backend) Fix parallel test execution issues +- ✨(backend) Add OPENSEARCH_INDEX_PREFIX setting to prevent naming overlaping + issues if the opensearch database is shared between apps. diff --git a/src/backend/core/management/commands/reindex_with_embedding.py b/src/backend/core/management/commands/reindex_with_embedding.py index f52d9e2..b2bf328 100644 --- a/src/backend/core/management/commands/reindex_with_embedding.py +++ b/src/backend/core/management/commands/reindex_with_embedding.py @@ -9,6 +9,7 @@ from django.core.management.base import BaseCommand, CommandError from opensearchpy.exceptions import NotFoundError +from core.models import get_opensearch_index_name from core.services.opensearch import ( check_hybrid_search_enabled, embed_text, @@ -31,7 +32,7 @@ class Command(BaseCommand): def handle(self, *args, **options): """Launch the reindexing with embedding.""" - index_name = options["index_name"] + index_name = get_opensearch_index_name(options["index_name"]) if not check_hybrid_search_enabled(): raise CommandError("Hybrid search is not enabled or properly configured.") diff --git a/src/backend/core/models.py b/src/backend/core/models.py index 91bcae6..ee34c45 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -3,9 +3,11 @@ import secrets import string +from django.conf import settings from django.contrib.auth.models import AbstractUser from django.db import models from django.db.models.functions import Length +from django.utils.functional import cached_property from django.utils.text import slugify from django.utils.translation import gettext_lazy as _ @@ -13,6 +15,11 @@ models.CharField.register_lookup(Length) TOKEN_LENGTH = 50 +def get_opensearch_index_name(name: str): + """Returns the opensearch index for a service name""" + return f"{settings.OPENSEARCH_INDEX_PREFIX}-{name}" + + class User(AbstractUser): """User for the find application""" @@ -61,3 +68,8 @@ class Service(models.Model): ) token = "".join(secrets.choice(characters) for _ in range(TOKEN_LENGTH)) return token + + @cached_property + def index_name(self): + """Returns the opensearch index for the service""" + return get_opensearch_index_name(self.name) diff --git a/src/backend/core/tests/commands/test_reindex_with_embedding.py b/src/backend/core/tests/commands/test_reindex_with_embedding.py index 2441bf6..c0d92e0 100644 --- a/src/backend/core/tests/commands/test_reindex_with_embedding.py +++ b/src/backend/core/tests/commands/test_reindex_with_embedding.py @@ -15,12 +15,12 @@ from core.management.commands.reindex_with_embedding import ( from core.management.commands.reindex_with_embedding import ( reindex_with_embedding, ) +from core.models import get_opensearch_index_name from core.services.opensearch import check_hybrid_search_enabled, opensearch_client from core.tests.mock import albert_embedding_response from core.tests.utils import ( bulk_create_documents, delete_search_pipeline, - delete_test_indices, enable_hybrid_search, prepare_index, ) @@ -43,7 +43,6 @@ def clear_caches(): # is different and must be cleared separately check_hybrid_search_enabled_command.cache_clear() delete_search_pipeline() - delete_test_indices() @responses.activate @@ -58,11 +57,12 @@ def test_reindex_with_embedding_command(settings): {"title": "cat", "content": "cats are curious and independent pets"}, ] ) - prepare_index(SERVICE_NAME, documents) + index_name = get_opensearch_index_name(SERVICE_NAME) + prepare_index(index_name, documents) # the index has not been embedded in the initial state initial_index = opensearch_client_.search( - index=SERVICE_NAME, size=3, body={"query": {"match_all": {}}} + index=index_name, size=3, body={"query": {"match_all": {}}} ) assert len(initial_index["hits"]["hits"]) == 3 for embedded_hit in initial_index["hits"]["hits"]: @@ -81,9 +81,9 @@ def test_reindex_with_embedding_command(settings): call_command("reindex_with_embedding", SERVICE_NAME) - opensearch_client_.indices.refresh(index=SERVICE_NAME) + opensearch_client_.indices.refresh(index=index_name) embedded_index = opensearch_client_.search( - index=SERVICE_NAME, size=3, body={"query": {"match_all": {}}} + index=index_name, size=3, body={"query": {"match_all": {}}} ) # the source index has been replaced with embedding version @@ -121,7 +121,8 @@ def test_reindex_can_fail_and_restart(settings): {"title": "cat", "content": "cats are curious and independent pets"}, ] ) - prepare_index(SERVICE_NAME, documents) + index_name = get_opensearch_index_name(SERVICE_NAME) + prepare_index(index_name, documents) # enable hybrid search after first indexing enable_hybrid_search(settings) @@ -147,16 +148,16 @@ def test_reindex_can_fail_and_restart(settings): status=200, ) - result = reindex_with_embedding(SERVICE_NAME) + result = reindex_with_embedding(index_name) # assert results reflect 2 successes and 1 failure assert result["nb_success_embedding"] == 2 assert result["nb_failed_embedding"] == 1 # assert the index state - opensearch_client_.indices.refresh(index=SERVICE_NAME) + opensearch_client_.indices.refresh(index=index_name) embedded_index = opensearch_client_.search( - index=SERVICE_NAME, size=3, body={"query": {"match_all": {}}} + index=index_name, size=3, body={"query": {"match_all": {}}} ) # Should have 2 documents with embeddings, 1 without due to error embedded_count = 0 @@ -180,16 +181,16 @@ def test_reindex_can_fail_and_restart(settings): json=albert_embedding_response.response, status=200, ) - result = reindex_with_embedding(SERVICE_NAME) + result = reindex_with_embedding(index_name) # assert results assert result["nb_success_embedding"] == 1 assert result["nb_failed_embedding"] == 0 # assert there is now 1 more success and 0 failures - opensearch_client_.indices.refresh(index=SERVICE_NAME) + opensearch_client_.indices.refresh(index=index_name) embedded_index = opensearch_client_.search( - index=SERVICE_NAME, size=3, body={"query": {"match_all": {}}} + index=index_name, size=3, body={"query": {"match_all": {}}} ) for hit in embedded_index["hits"]["hits"]: assert ( @@ -220,7 +221,8 @@ def test_reindex_preserves_concurrent_updates(settings): {"title": "dog", "content": "dogs are loyal domestic animals"}, ] ) - prepare_index(SERVICE_NAME, documents) + index_name = get_opensearch_index_name(SERVICE_NAME) + prepare_index(index_name, documents) enable_hybrid_search(settings) updated_title = "updated dog" @@ -231,7 +233,7 @@ def test_reindex_preserves_concurrent_updates(settings): patch( "core.services.opensearch.opensearch_client_.search", side_effect=opensearch_client_.update( - index=SERVICE_NAME, + index=index_name, id=documents[1]["id"], body={ "doc": { @@ -249,13 +251,13 @@ def test_reindex_preserves_concurrent_updates(settings): json=albert_embedding_response.response, status=200, ) - result = reindex_with_embedding(SERVICE_NAME) + result = reindex_with_embedding(index_name) assert result["nb_success_embedding"] == 2 assert result["nb_failed_embedding"] == 0 - opensearch_client_.indices.refresh(index=SERVICE_NAME) + opensearch_client_.indices.refresh(index=index_name) embedded_index = opensearch_client_.search( - index=SERVICE_NAME, size=2, body={"query": {"match_all": {}}} + index=index_name, size=2, body={"query": {"match_all": {}}} ) # Check that the latest update is preserved dog_doc = [ @@ -281,7 +283,9 @@ def test_reindex_command_but_index_does_not_exist(settings): wrong_index = "wrong-index-name" enable_hybrid_search(settings) + wrong_index_name = get_opensearch_index_name(wrong_index) + with pytest.raises(CommandError) as err: call_command("reindex_with_embedding", wrong_index) - assert str(err.value) == f"Index {wrong_index} does not exist." + assert str(err.value) == f"Index {wrong_index_name} does not exist." diff --git a/src/backend/core/tests/conftest.py b/src/backend/core/tests/conftest.py index 083d791..31b37bf 100644 --- a/src/backend/core/tests/conftest.py +++ b/src/backend/core/tests/conftest.py @@ -1,10 +1,15 @@ """Fixtures for tests in the find core application""" import pytest +from faker import Faker from lasuite.oidc_resource_server.authentication import ( get_resource_server_backend, ) +from core.services import opensearch + +fake = Faker() + @pytest.fixture(name="jwt_rs_backend") def jwt_resource_server_backend_fixture(settings): @@ -20,3 +25,27 @@ def jwt_resource_server_backend_fixture(settings): settings.OIDC_RS_BACKEND_CLASS = _original_backend get_resource_server_backend.cache_clear() + + +@pytest.fixture(autouse=True) +def cleanup_test_index(settings): + """ + Fixture to set a randomized prefix for all service indexes within the tests + and remove them on tear down. + """ + _original_prefix = settings.OPENSEARCH_INDEX_PREFIX + prefix = "".join(fake.random_letters(5)).lower() + settings.OPENSEARCH_INDEX_PREFIX = prefix + + # Create client here to prevent "teardown" issues when the opensearch settings are + # removed for error tests. + client = opensearch.opensearch_client() + + yield + + settings.OPENSEARCH_INDEX_PREFIX = _original_prefix + + try: + client.indices.delete(index=f"{prefix}-*") + except opensearch.NotFoundError: + pass diff --git a/src/backend/core/tests/test_api_documents_index_bulk.py b/src/backend/core/tests/test_api_documents_index_bulk.py index c59d6ef..d1578a6 100644 --- a/src/backend/core/tests/test_api_documents_index_bulk.py +++ b/src/backend/core/tests/test_api_documents_index_bulk.py @@ -43,7 +43,7 @@ def test_api_documents_index_bulk_invalid_token(): def test_api_documents_index_bulk_success(): """A registered service should be able to index documents in bulk with a valid token.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) response = APIClient().post( @@ -61,14 +61,11 @@ def test_api_documents_index_bulk_success(): def test_api_documents_index_bulk_ensure_index(): """A registered service should be create the opensearch index if need.""" opensearch_client_ = opensearch.opensearch_client() - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) - # Delete the index - opensearch_client_.indices.delete(index="*test*") - with pytest.raises(opensearch.NotFoundError): - opensearch_client_.indices.get(index="test-service") + opensearch_client_.indices.get(index=service.index_name) response = APIClient().post( "/api/v1.0/documents/index/", @@ -83,7 +80,7 @@ def test_api_documents_index_bulk_ensure_index(): assert [d["status"] for d in responses] == ["success"] * 3 # The index has been rebuilt - opensearch_client_.indices.get(index="test-service") + opensearch_client_.indices.get(index=service.index_name) @pytest.mark.parametrize( @@ -202,7 +199,7 @@ def test_api_documents_index_bulk_invalid_document( field, invalid_value, error_type, error_message ): """Test bulk document indexing with various invalid fields.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) # Modify the first document with the invalid value for the specified field @@ -242,7 +239,7 @@ def test_api_documents_index_bulk_invalid_document( ) def test_api_documents_index_bulk_required(field): """Test bulk document indexing with a required field missing.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) del documents[0][field] @@ -274,7 +271,7 @@ def test_api_documents_index_bulk_required(field): ) def test_api_documents_index_bulk_default(field, default_value): """Test bulk document indexing while removing optional fields that have default values.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) del documents[0][field] @@ -291,14 +288,14 @@ def test_api_documents_index_bulk_default(field, default_value): assert [d["status"] for d in responses] == ["success"] * 3 indexed_document = opensearch.opensearch_client().get( - index=service.name, id=responses[0]["_id"] + index=service.index_name, id=responses[0]["_id"] )["_source"] assert indexed_document[field] == default_value def test_api_documents_index_bulk_updated_at_before_created_at(): """Test bulk document indexing with updated_at before created_at.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) documents[0]["updated_at"] = documents[0]["created_at"] - datetime.timedelta( @@ -332,7 +329,7 @@ def test_api_documents_index_bulk_updated_at_before_created_at(): ) def test_api_documents_index_bulk_datetime_future(field): """Test bulk document indexing with datetimes in the future.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) now = timezone.now() @@ -360,7 +357,7 @@ def test_api_documents_index_bulk_datetime_future(field): def test_api_documents_index_empty_content_check(): """Test bulk document indexing with both empty title & content.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) documents[0]["content"] = "" @@ -388,7 +385,7 @@ def test_api_documents_index_empty_content_check(): def test_api_documents_index_opensearch_errors(): """Test bulk document indexing errors""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch(3) with mock.patch.object(opensearch.opensearch_client(), "bulk") as mock_bulk: diff --git a/src/backend/core/tests/test_api_documents_index_single.py b/src/backend/core/tests/test_api_documents_index_single.py index 219446e..72ba66d 100644 --- a/src/backend/core/tests/test_api_documents_index_single.py +++ b/src/backend/core/tests/test_api_documents_index_single.py @@ -11,7 +11,7 @@ from rest_framework.test import APIClient from core import factories from core.services import opensearch from core.tests.mock import albert_embedding_response -from core.tests.utils import delete_test_indices, enable_hybrid_search +from core.tests.utils import enable_hybrid_search pytestmark = pytest.mark.django_db @@ -56,7 +56,7 @@ def test_api_documents_index_single_hybrid_enabled_success(settings): If hybrid search is enabled, the indexing should have embedding of dimension settings.EMBEDDING_DIMENSION. """ - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() enable_hybrid_search(settings) responses.add( responses.POST, @@ -64,6 +64,7 @@ def test_api_documents_index_single_hybrid_enabled_success(settings): json=albert_embedding_response.response, status=200, ) + document = factories.DocumentSchemaFactory.build() response = APIClient().post( @@ -77,7 +78,7 @@ def test_api_documents_index_single_hybrid_enabled_success(settings): assert response.json()["_id"] == str(document["id"]) new_indexed_document = opensearch.opensearch_client().get( - index=service.name, id=str(document["id"]) + index=service.index_name, id=str(document["id"]) ) assert new_indexed_document["_version"] == 1 assert new_indexed_document["_source"]["title"] == document["title"].strip().lower() @@ -94,7 +95,7 @@ def test_api_documents_index_single_hybrid_enabled_success(settings): def test_api_documents_index_single_hybrid_disabled_success(): """If hybrid search is not enabled, the indexing should have an embedding equal to None.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() document = factories.DocumentSchemaFactory.build() opensearch.check_hybrid_search_enabled.cache_clear() @@ -109,7 +110,7 @@ def test_api_documents_index_single_hybrid_disabled_success(): assert response.json()["_id"] == str(document["id"]) new_indexed_document = opensearch.opensearch_client().get( - index=service.name, id=str(document["id"]) + index=service.index_name, id=str(document["id"]) ) assert new_indexed_document["_version"] == 1 assert new_indexed_document["_source"]["title"] == document["title"].strip().lower() @@ -117,15 +118,14 @@ def test_api_documents_index_single_hybrid_disabled_success(): assert new_indexed_document["_source"]["embedding"] is None -def test_api_documents_index_bulk_ensure_index(settings): +def test_api_documents_index_single_ensure_index(settings): """A registered service should be create the opensearch index if need.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() document = factories.DocumentSchemaFactory.build() opensearch_client_ = opensearch.opensearch_client() - delete_test_indices() with pytest.raises(opensearch.NotFoundError): - opensearch_client_.indices.get(index="test-service") + opensearch_client_.indices.get(index=service.index_name) response = APIClient().post( "/api/v1.0/documents/index/", @@ -138,9 +138,9 @@ def test_api_documents_index_bulk_ensure_index(settings): assert response.json()["_id"] == str(document["id"]) # The index has been rebuilt - data = opensearch_client_.indices.get(index="test-service") + data = opensearch_client_.indices.get(index=service.index_name) - assert data["test-service"]["mappings"] == { + assert data[service.index_name]["mappings"] == { "dynamic": "strict", "properties": { "id": {"type": "keyword"}, @@ -295,7 +295,7 @@ def test_api_documents_index_single_invalid_document( field, invalid_value, error_type, error_message ): """Test document indexing with various invalid fields.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() document = factories.DocumentSchemaFactory.build() # Modify the document with the invalid value for the specified field @@ -330,7 +330,7 @@ def test_api_documents_index_single_invalid_document( ) def test_api_documents_index_single_required(field): """Test document indexing with a required field missing.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() document = factories.DocumentSchemaFactory.build() del document[field] @@ -357,7 +357,7 @@ def test_api_documents_index_single_required(field): ) def test_api_documents_index_single_default(field, default_value): """Test document indexing while removing optional fields that have default values.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() document = factories.DocumentSchemaFactory.build() del document[field] @@ -373,14 +373,14 @@ def test_api_documents_index_single_default(field, default_value): assert response.json()["_id"] == str(document["id"]) indexed_document = opensearch.opensearch_client().get( - index=service.name, id=str(document["id"]) + index=service.index_name, id=str(document["id"]) )["_source"] assert indexed_document[field] == default_value def test_api_documents_index_single_udpated_at_before_created(): """Test document indexing with updated_at before created_at.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() document = factories.DocumentSchemaFactory.build() document["updated_at"] = document["created_at"] - datetime.timedelta(seconds=1) @@ -406,7 +406,7 @@ def test_api_documents_index_single_udpated_at_before_created(): ) def test_api_documents_index_single_datetime_future(field): """Test document indexing with datetimes in the future.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() document = factories.DocumentSchemaFactory.build() now = timezone.now() @@ -426,7 +426,7 @@ def test_api_documents_index_single_datetime_future(field): def test_api_documents_index_empty_content_check(): """Test document indexing with both empty title & content.""" - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() document = factories.DocumentSchemaFactory.build() document["content"] = "" diff --git a/src/backend/core/tests/test_api_documents_search.py b/src/backend/core/tests/test_api_documents_search.py index 19c0905..b6a01a2 100644 --- a/src/backend/core/tests/test_api_documents_search.py +++ b/src/backend/core/tests/test_api_documents_search.py @@ -49,8 +49,8 @@ def test_api_documents_search_auth_invalid_parameters(settings): settings.OIDC_RS_CLIENT_ID = None settings.OIDC_RS_CLIENT_SECRET = None - service = factories.ServiceFactory(name="test-service") - prepare_index(service.name, []) + service = factories.ServiceFactory() + prepare_index(service.index_name, []) response = APIClient().post( "/api/v1.0/documents/search/", @@ -70,7 +70,7 @@ def test_api_documents_search_opensearch_env_variables_not_set(settings): result in a 500 internal server error """ setup_oicd_resource_server(responses, settings, sub="user_sub") - factories.ServiceFactory(name="test-service") + factories.ServiceFactory() del settings.OPENSEARCH_HOST # Remove required settings del settings.OPENSEARCH_PASSWORD @@ -100,8 +100,8 @@ def test_api_documents_search_query_unknown_user(settings): token = build_authorization_bearer() - service = factories.ServiceFactory(name="test-service") - prepare_index(service.name, []) + service = factories.ServiceFactory() + prepare_index(service.index_name, []) response = APIClient().post( "/api/v1.0/documents/search/", @@ -124,7 +124,7 @@ def test_api_documents_search_services_invalid_parameters(settings): json=albert_embedding_response.response, status=200, ) - factories.ServiceFactory(name="test-service") + factories.ServiceFactory() response = APIClient().post( "/api/v1.0/documents/search/", @@ -154,7 +154,7 @@ def test_api_documents_search_reached_docs_invalid_parameters(settings): json=albert_embedding_response.response, status=200, ) - factories.ServiceFactory(name="test-service") + factories.ServiceFactory() response = APIClient().post( "/api/v1.0/documents/search/", @@ -186,11 +186,11 @@ def test_api_documents_search_match_all(settings): status=200, ) nb_documents = 12 - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch( nb_documents, reach=random.choice(["public", "authenticated"]) ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) response = APIClient().post( "/api/v1.0/documents/search/", @@ -209,7 +209,7 @@ def test_api_documents_search_match_all(settings): def test_api_documents_full_text_search_query_title(settings): """Searching a document by its title should work as expected""" setup_oicd_resource_server(responses, settings, sub="user_sub") - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = bulk_create_documents( [ @@ -218,7 +218,8 @@ def test_api_documents_full_text_search_query_title(settings): {"title": "The brown goat", "content": "the wolf"}, ] ) - prepare_index(service.name, documents) + + prepare_index(service.index_name, documents) response = APIClient().post( "/api/v1.0/documents/search/", @@ -278,7 +279,7 @@ def test_api_documents_full_text_search(settings): setup_oicd_resource_server(responses, settings, sub="user_sub") token = build_authorization_bearer() - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = bulk_create_documents( [ {"title": "The quick brown fox", "content": "the wolf"}, @@ -286,7 +287,7 @@ def test_api_documents_full_text_search(settings): {"title": "The brown goat", "content": "the wolf"}, ] ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) response = APIClient().post( "/api/v1.0/documents/search/", @@ -356,7 +357,7 @@ def test_api_documents_hybrid_search(settings): status=200, ) # mock embedding API - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = bulk_create_documents( [ {"title": "The quick brown fox", "content": "the wolf"}, @@ -364,7 +365,7 @@ def test_api_documents_hybrid_search(settings): {"title": "The brown goat", "content": "the wolf"}, ] ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) response = APIClient().post( "/api/v1.0/documents/search/", @@ -458,11 +459,11 @@ def test_api_documents_search_ordering_by_fields(settings): json=albert_embedding_response.response, status=200, ) - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch( 4, reach=random.choice(["public", "authenticated"]) ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) parameters = [ (enums.TITLE, "asc"), @@ -511,11 +512,11 @@ def test_api_documents_search_ordering_by_relevance(settings): json=albert_embedding_response.response, status=200, ) - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch( 4, reach=random.choice(["public", "authenticated"]) ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) for direction in ["asc", "desc"]: response = APIClient().post( @@ -552,11 +553,11 @@ def test_api_documents_search_ordering_by_unknown_field(settings): status=200, ) # Setup: Initialize the service and documents only once - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch( 2, reach=random.choice(["public", "authenticated"]) ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) # Define the parameters manually directions = ["asc", "desc"] @@ -599,11 +600,11 @@ def test_api_documents_search_ordering_by_unknown_direction(settings): json=albert_embedding_response.response, status=200, ) - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch( 2, reach=random.choice(["public", "authenticated"]) ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) for field in enums.ORDER_BY_OPTIONS: response = APIClient().post( @@ -639,11 +640,11 @@ def test_api_documents_search_filtering_by_reach(settings): json=albert_embedding_response.response, status=200, ) - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch( 4, reach=random.choice(["public", "authenticated"]) ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) for reach in enums.ReachEnum: response = APIClient().post( @@ -675,12 +676,12 @@ def test_api_documents_search_with_nb_results(settings): json=albert_embedding_response.response, status=200, ) - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch( 9, reach=random.choice(["public", "authenticated"]) ) ids = [str(doc["id"]) for doc in documents] - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) nb_results = 3 response = APIClient().post( @@ -741,11 +742,11 @@ def test_api_documents_search_nb_results_invalid_parameters(settings): json=albert_embedding_response.response, status=200, ) - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = factories.DocumentSchemaFactory.build_batch( 4, reach=random.choice(["public", "authenticated"]) ) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) parameters = [ ( @@ -782,13 +783,13 @@ def test_api_documents_search_nb_results_with_filtering(settings): json=albert_embedding_response.response, status=200, ) - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() public_documents = factories.DocumentSchemaFactory.build_batch(3, reach="public") public_ids = [str(doc["id"]) for doc in public_documents] private_documents = factories.DocumentSchemaFactory.build_batch( 2, reach="authenticated" ) - prepare_index(service.name, public_documents + private_documents) + prepare_index(service.index_name, public_documents + private_documents) nb_results = 3 response = APIClient().post( diff --git a/src/backend/core/tests/test_api_documents_search_access_control.py b/src/backend/core/tests/test_api_documents_search_access_control.py index 27a9082..96d741c 100644 --- a/src/backend/core/tests/test_api_documents_search_access_control.py +++ b/src/backend/core/tests/test_api_documents_search_access_control.py @@ -15,7 +15,6 @@ from core.services.opensearch import opensearch_client from .mock import albert_embedding_response from .utils import ( build_authorization_bearer, - delete_test_indices, prepare_index, setup_oicd_resource_server, ) @@ -32,11 +31,11 @@ def test_api_documents_search_access_control_anonymous(settings): json=albert_embedding_response.response, status=200, ) - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents = [] for reach in enums.ReachEnum: documents.extend(factories.DocumentSchemaFactory.build_batch(3, reach=reach)) - prepare_index(service.name, documents) + prepare_index(service.index_name, documents) response = APIClient().post("/api/v1.0/documents/search/?q=*") @@ -61,7 +60,7 @@ def test_api_documents_search_access_control(settings): setup_oicd_resource_server(responses, settings, sub="user_sub") token = build_authorization_bearer() - service = factories.ServiceFactory(name="test-service") + service = factories.ServiceFactory() documents_reach = factories.DocumentSchemaFactory.build_batch(6) documents_open = [ doc for doc in documents_reach if doc["reach"] in ["authenticated", "public"] @@ -71,7 +70,7 @@ def test_api_documents_search_access_control(settings): ) expected_ids = [doc["id"] for doc in documents_open + documents_user] - prepare_index(service.name, documents_user + documents_reach) + prepare_index(service.index_name, documents_user + documents_reach) response = APIClient().post( "/api/v1.0/documents/search/", @@ -114,7 +113,7 @@ def test_api_documents_search_access__only_visited_public( setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs") token = build_authorization_bearer() - service = factories.ServiceFactory(name="test-service", client_id="docs") + service = factories.ServiceFactory(client_id="docs") docs = [ factories.DocumentSchemaFactory( @@ -123,7 +122,7 @@ def test_api_documents_search_access__only_visited_public( for doc_id in doc_ids ] - prepare_index(service.name, docs) + prepare_index(service.index_name, docs) response = APIClient().post( "/api/v1.0/documents/search/", @@ -151,7 +150,7 @@ def test_api_documents_search_access__any_owner_public(settings): setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs") token = build_authorization_bearer() - service = factories.ServiceFactory(name="test-service", client_id="docs") + service = factories.ServiceFactory(client_id="docs") docs = factories.DocumentSchemaFactory.build_batch( 6, @@ -165,7 +164,7 @@ def test_api_documents_search_access__any_owner_public(settings): users=["other_sub"], ) - prepare_index(service.name, docs + other_docs) + prepare_index(service.index_name, docs + other_docs) expected = [d["id"] for d in docs] @@ -195,8 +194,8 @@ def test_api_documents_search_access__services(settings): setup_oicd_resource_server(responses, settings, sub="user_sub", audience="a-client") token = build_authorization_bearer() - service_a = factories.ServiceFactory(name="test-index-a", client_id="a-client") - service_b = factories.ServiceFactory(name="test-index-b", client_id="b-client") + service_a = factories.ServiceFactory(client_id="a-client") + service_b = factories.ServiceFactory(client_id="b-client") service_a_docs = factories.DocumentSchemaFactory.build_batch( 3, reach=enums.ReachEnum.AUTHENTICATED, users=["user_sub"] @@ -207,8 +206,8 @@ def test_api_documents_search_access__services(settings): expected_ids = [doc["id"] for doc in service_a_docs] - prepare_index(service_a.name, service_a_docs) - prepare_index(service_b.name, service_b_docs, cleanup=False) + prepare_index(service_a.index_name, service_a_docs) + prepare_index(service_b.index_name, service_b_docs) response = APIClient().post( "/api/v1.0/documents/search/", @@ -236,7 +235,6 @@ def test_api_documents_search_access__missing_index(settings): token = build_authorization_bearer() factories.ServiceFactory(name="test-index-a", client_id="a-client") - delete_test_indices() opensearch_client.cache_clear() # a-client has no index. ignore it. @@ -284,9 +282,9 @@ def test_api_documents_search_access__related_services(settings): expected_ids = [doc["id"] for doc in service_a_docs + service_c_docs] - prepare_index(service_a.name, service_a_docs) - prepare_index(service_b.name, service_b_docs, cleanup=False) - prepare_index(service_c.name, service_c_docs, cleanup=False) + prepare_index(service_a.index_name, service_a_docs) + prepare_index(service_b.index_name, service_b_docs) + prepare_index(service_c.index_name, service_c_docs) response = APIClient().post( "/api/v1.0/documents/search/", @@ -328,8 +326,8 @@ def test_api_documents_search_access__related_missing_index(settings): expected_ids = [doc["id"] for doc in service_c_docs] - prepare_index(service_b.name, service_b_docs) - prepare_index(service_c.name, service_c_docs, cleanup=False) + prepare_index(service_b.index_name, service_b_docs) + prepare_index(service_c.index_name, service_c_docs) # a-client has no index. ignore it. response = APIClient().post( @@ -375,9 +373,9 @@ def test_api_documents_search_access__request_services(settings): expected_ids = [doc["id"] for doc in service_c_docs] - prepare_index(service_a.name, service_a_docs) - prepare_index(service_b.name, service_b_docs, cleanup=False) - prepare_index(service_c.name, service_c_docs, cleanup=False) + prepare_index(service_a.index_name, service_a_docs) + prepare_index(service_b.index_name, service_b_docs) + prepare_index(service_c.index_name, service_c_docs) response = APIClient().post( "/api/v1.0/documents/search/", @@ -453,7 +451,7 @@ def test_api_documents_search_access__authenticated(settings): setup_oicd_resource_server(responses, settings, sub="user_sub", audience="docs") token = build_authorization_bearer() - service = factories.ServiceFactory(name="test-service", client_id="docs") + service = factories.ServiceFactory(client_id="docs") documents_open = factories.DocumentSchemaFactory.build_batch( 2, reach=enums.ReachEnum.PUBLIC @@ -470,7 +468,9 @@ def test_api_documents_search_access__authenticated(settings): ) documents = documents_user + documents_open + documents_restricted - prepare_index(service.name, documents_user + documents_open + documents_restricted) + prepare_index( + service.index_name, documents_user + documents_open + documents_restricted + ) # Only owned documents (reach is ignored) response = APIClient().post( diff --git a/src/backend/core/tests/test_models_services.py b/src/backend/core/tests/test_models_services.py index f7bf6e3..b7dfcb9 100644 --- a/src/backend/core/tests/test_models_services.py +++ b/src/backend/core/tests/test_models_services.py @@ -17,10 +17,11 @@ def test_models_services_name_unique(): factories.ServiceFactory(name=service.name) -def test_models_services_name_slugified(): +def test_models_services_name_slugified(settings): """The name field should be slugified.""" service = factories.ServiceFactory(name="My service name") assert service.name == "my-service-name" + assert service.index_name == f"{settings.OPENSEARCH_INDEX_PREFIX}-my-service-name" def test_models_services_token_50_characters_exact(): diff --git a/src/backend/core/tests/test_opensearch.py b/src/backend/core/tests/test_opensearch.py index b72169e..cf988a9 100644 --- a/src/backend/core/tests/test_opensearch.py +++ b/src/backend/core/tests/test_opensearch.py @@ -11,6 +11,7 @@ import responses from core.services import opensearch +from .. import factories from ..services.opensearch import ( check_hybrid_search_enabled, embed_text, @@ -170,7 +171,8 @@ def test_fall_back_on_full_text_search_if_hybrid_search_disabled(settings, caplo {"title": "cat", "content": "cats are curious and independent pets"}, ] ) - prepare_index(SERVICE_NAME, documents) + service = factories.ServiceFactory(name=SERVICE_NAME) + prepare_index(service.index_name, documents) q = "wolf" with caplog.at_level(logging.INFO): @@ -207,7 +209,8 @@ def test_fall_back_on_full_text_search_if_embedding_api_fails(settings, caplog): {"title": "cat", "content": "cats are curious and independent pets"}, ] ) - prepare_index(SERVICE_NAME, documents) + service = factories.ServiceFactory(name=SERVICE_NAME) + prepare_index(service.index_name, documents) q = "wolf" with caplog.at_level(logging.INFO): @@ -238,7 +241,8 @@ def test_fall_back_on_full_text_search_if_variable_are_missing(settings, caplog) {"title": "cat", "content": "cats are curious and independent pets"}, ] ) - prepare_index(SERVICE_NAME, documents) + service = factories.ServiceFactory(name=SERVICE_NAME) + prepare_index(service.index_name, documents) q = "wolf" with caplog.at_level(logging.INFO): @@ -274,7 +278,8 @@ def test_match_all(settings, caplog): {"title": "cat", "content": "cats are curious and independent pets"}, ] ) - prepare_index(SERVICE_NAME, documents) + service = factories.ServiceFactory(name=SERVICE_NAME) + prepare_index(service.index_name, documents) q = "*" with caplog.at_level(logging.INFO): @@ -304,7 +309,8 @@ def test_search_ordering_by_relevance(settings, caplog): ] ) q = "canine pet" - prepare_index(SERVICE_NAME, documents) + service = factories.ServiceFactory(name=SERVICE_NAME) + prepare_index(service.index_name, documents) for direction in ["asc", "desc"]: with caplog.at_level(logging.INFO): @@ -338,7 +344,8 @@ def test_hybrid_search_number_of_matches(settings): {"title": "cat", "content": "cats are curious and independent pets"}, ] ) - prepare_index(SERVICE_NAME, documents) + service = factories.ServiceFactory(name=SERVICE_NAME) + prepare_index(service.index_name, documents) q = "pony" # full-text matches 0 document for nb_results in [1, 2, 3]: # semantic should match k documents diff --git a/src/backend/core/tests/utils.py b/src/backend/core/tests/utils.py index 32fa2e9..9a21f3a 100644 --- a/src/backend/core/tests/utils.py +++ b/src/backend/core/tests/utils.py @@ -58,16 +58,8 @@ def delete_search_pipeline(): logger.info("Search pipeline not found, nothing to delete.") -def delete_test_indices(): - """Drop all search index containing the 'test' word""" - opensearch.opensearch_client().indices.delete(index="*test*") - - -def prepare_index(index_name, documents: List, cleanup=True): +def prepare_index(index_name, documents: List): """Prepare the search index before testing a query on it.""" - if cleanup: - delete_test_indices() - opensearch.ensure_index_exists(index_name) # Index new documents diff --git a/src/backend/core/views.py b/src/backend/core/views.py index 05601f1..82b84ca 100644 --- a/src/backend/core/views.py +++ b/src/backend/core/views.py @@ -13,7 +13,7 @@ from rest_framework.response import Response from . import schemas from .authentication import ServiceTokenAuthentication -from .models import Service +from .models import Service, get_opensearch_index_name from .permissions import IsAuthAuthenticated from .services.opensearch import ( check_hybrid_search_enabled, @@ -86,7 +86,7 @@ class IndexDocumentView(views.APIView): - Returns a list of results for all documents, with details of success and indexing errors. """ - index_name = request.auth.name + index_name = request.auth.index_name opensearch_client_ = opensearch_client() if isinstance(request.data, list): @@ -191,7 +191,7 @@ class SearchDocumentView(ResourceServerMixin, views.APIView): if len(available) < len(services): raise SuspiciousOperation("Some requested services are not available") - return allowed_services + return [get_opensearch_index_name(name) for name in allowed_services] def post(self, request, *args, **kwargs): """ diff --git a/src/backend/find/settings.py b/src/backend/find/settings.py index ac39ef2..8c26ea4 100755 --- a/src/backend/find/settings.py +++ b/src/backend/find/settings.py @@ -243,6 +243,9 @@ class Base(Configuration): OPENSEARCH_USE_SSL = values.BooleanValue( default=True, environ_name="OPENSEARCH_USE_SSL", environ_prefix=None ) + OPENSEARCH_INDEX_PREFIX = values.Value( + default="find", environ_name="OPENSEARCH_INDEX_PREFIX", environ_prefix=None + ) SPECTACULAR_SETTINGS = { "TITLE": "Find API",