electron-rare/find
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨(schema) add fields to the document

Browse Source 
We need to index the tree structure information as well as an
active field that can be set to False when the item is deleted on
the remote service. We could delete the item from our search index
but it is safer to keep all documents synchronized and not only
those which are not deleted.
This commit is contained in:
Samuel Paccoud - DINUM
2025-08-07 17:42:02 +02:00
committed by Florian Fabre
parent 0b472720bf
commit 64687d3045
8 changed files with 136 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backend/core/enums.py
+6 -1
View File
@@ -16,12 +16,17 @@ class ReachEnum(str, Enum):
# Fields
CREATED_AT = "created_at"
DEPTH = "depth"
PATH = "path"
NUMCHILD = "numchild"
REACH = "reach"
SIZE = "size"
TITLE = "title"
UPDATED_AT = "updated_at"
USERS = "users"
GROUPS = "groups"
RELEVANCE = "relevance"
ORDER_BY_OPTIONS = (RELEVANCE, TITLE, CREATED_AT, UPDATED_AT, SIZE, REACH)
SOURCE_FIELDS = (TITLE, SIZE, CREATED_AT, UPDATED_AT, REACH)
SOURCE_FIELDS = (TITLE, SIZE, DEPTH, PATH, NUMCHILD, CREATED_AT, UPDATED_AT, REACH)
src/backend/core/factories.py
+4
View File
@@ -21,6 +21,7 @@ class DocumentSchemaFactory(factory.DictFactory):
id = factory.LazyFunction(uuid4)
title = factory.Sequence(lambda n: f"Test title {n!s}")
path = factory.Sequence(lambda n: f"000{n}")
content = factory.Sequence(lambda n: f"Test content {n!s}")
created_at = factory.LazyFunction(
lambda: fake.date_time_this_decade(tzinfo=timezone.get_current_timezone())
@@ -29,6 +30,9 @@ class DocumentSchemaFactory(factory.DictFactory):
users = factory.LazyFunction(lambda: [uuid4() for _ in range(3)])
groups = factory.LazyFunction(lambda: [slugify(fake.word()) for _ in range(3)])
reach = factory.Iterator(list(enums.ReachEnum))
depth = 1
numchild = 0
is_active = True
@factory.lazy_attribute
def updated_at(self):
src/backend/core/opensearch.py
+10 -2
View File
@@ -25,14 +25,21 @@ def ensure_index_exists(index_name):
"mappings": {
"dynamic": "strict",
"properties": {
"id": {"type": "keyword"},
"title": {
"type": "keyword", # Primary field for exact matches and sorting
"fields": {
"text": {
"type": "text" # Sub-field for full-text search
}
"type": "text"
} # Sub-field for full-text search
},
},
"depth": {"type": "integer"},
"path": {
"type": "keyword",
"fields": {"text": {"type": "text"}},
},
"numchild": {"type": "integer"},
"content": {"type": "text"},
"created_at": {"type": "date"},
"updated_at": {"type": "date"},
@@ -40,6 +47,7 @@ def ensure_index_exists(index_name):
"users": {"type": "keyword"},
"groups": {"type": "keyword"},
"reach": {"type": "keyword"},
"is_active": {"type": "boolean"},
},
}
},
src/backend/core/schemas.py
+4
View File
@@ -24,6 +24,9 @@ class DocumentSchema(BaseModel):
id: UUID4
title: Annotated[str, Field(max_length=300)]
depth: Annotated[int, Field(ge=0)]
path: Annotated[str, Field(max_length=300)]
numchild: Annotated[int, Field(ge=0)]
content: str
created_at: AwareDatetime
updated_at: AwareDatetime
@@ -33,6 +36,7 @@ class DocumentSchema(BaseModel):
default_factory=list
)
reach: Optional[enums.ReachEnum] = Field(default=enums.ReachEnum.RESTRICTED)
is_active: bool
model_config = ConfigDict(
str_min_length=1, str_strip_whitespace=True, use_enum_values=True
src/backend/core/tests/test_api_documents_index_bulk.py
+49 -1
View File
@@ -77,6 +77,36 @@ def test_api_documents_index_bulk_success():
"string_too_long",
"String should have at most 300 characters",
),
(
"depth",
-1,
"greater_than_equal",
"Input should be greater than or equal to 0",
),
(
"depth",
"a",
"int_parsing",
"Input should be a valid integer, unable to parse string as an integer",
),
(
"path",
"a" * 301,
"string_too_long",
"String should have at most 300 characters",
),
(
"numchild",
-1,
"greater_than_equal",
"Input should be greater than or equal to 0",
),
(
"numchild",
"a",
"int_parsing",
"Input should be a valid integer, unable to parse string as an integer",
),
("content", 1, "string_type", "Input should be a valid string"),
(
"created_at",
@@ -135,6 +165,12 @@ def test_api_documents_index_bulk_success():
"enum",
"Input should be 'public', 'authenticated' or 'restricted'",
),
(
"is_active",
"invalid",
"bool_parsing",
"Input should be a valid boolean, unable to interpret input",
),
],
)
def test_api_documents_index_bulk_invalid_document(
@@ -167,7 +203,19 @@ def test_api_documents_index_bulk_invalid_document(
@pytest.mark.parametrize(
"field", ["id", "title", "content", "size", "created_at", "updated_at"]
"field",
[
"id",
"title",
"depth",
"path",
"numchild",
"content",
"size",
"created_at",
"updated_at",
"is_active",
],
)
def test_api_documents_index_bulk_required(field):
"""Test bulk document indexing with a required field missing."""
src/backend/core/tests/test_api_documents_index_single.py
+49 -1
View File
@@ -74,6 +74,36 @@ def test_api_documents_index_single_success():
"string_too_long",
"String should have at most 300 characters",
),
(
"depth",
-1,
"greater_than_equal",
"Input should be greater than or equal to 0",
),
(
"depth",
"a",
"int_parsing",
"Input should be a valid integer, unable to parse string as an integer",
),
(
"path",
"a" * 301,
"string_too_long",
"String should have at most 300 characters",
),
(
"numchild",
-1,
"greater_than_equal",
"Input should be greater than or equal to 0",
),
(
"numchild",
"a",
"int_parsing",
"Input should be a valid integer, unable to parse string as an integer",
),
("content", 1, "string_type", "Input should be a valid string"),
(
"created_at",
@@ -132,6 +162,12 @@ def test_api_documents_index_single_success():
"enum",
"Input should be 'public', 'authenticated' or 'restricted'",
),
(
"is_active",
"invalid",
"bool_parsing",
"Input should be a valid boolean, unable to interpret input",
),
],
)
def test_api_documents_index_single_invalid_document(
@@ -157,7 +193,19 @@ def test_api_documents_index_single_invalid_document(
@pytest.mark.parametrize(
"field", ["id", "title", "content", "size", "created_at", "updated_at"]
"field",
[
"id",
"title",
"depth",
"path",
"numchild",
"content",
"size",
"created_at",
"updated_at",
"is_active",
],
)
def test_api_documents_index_single_required(field):
"""Test document indexing with a required field missing."""
src/backend/core/tests/test_api_documents_search.py
+12
View File
@@ -48,6 +48,9 @@ def test_api_documents_search_query_title():
assert list(fox_data.keys()) == ["_index", "_id", "_score", "_source", "fields"]
assert fox_data["_id"] == str(document["id"])
assert fox_data["_source"] == {
"depth": 1,
"numchild": 0,
"path": document["path"],
"size": document["size"],
"created_at": document["created_at"].isoformat(),
"updated_at": document["updated_at"].isoformat(),
@@ -66,6 +69,9 @@ def test_api_documents_search_query_title():
]
assert other_fox_data["_id"] == str(other_fox_document["id"])
assert other_fox_data["_source"] == {
"depth": 1,
"numchild": 0,
"path": other_fox_document["path"],
"size": other_fox_document["size"],
"created_at": other_fox_document["created_at"].isoformat(),
"updated_at": other_fox_document["updated_at"].isoformat(),
@@ -104,6 +110,9 @@ def test_api_documents_search_query_content():
assert list(fox_data.keys()) == ["_index", "_id", "_score", "_source", "fields"]
assert fox_data["_id"] == str(document["id"])
assert fox_data["_source"] == {
"depth": 1,
"numchild": 0,
"path": document["path"],
"size": document["size"],
"created_at": document["created_at"].isoformat(),
"updated_at": document["updated_at"].isoformat(),
@@ -122,6 +131,9 @@ def test_api_documents_search_query_content():
]
assert other_fox_data["_id"] == str(other_fox_document["id"])
assert other_fox_data["_source"] == {
"depth": 1,
"numchild": 0,
"path": other_fox_document["path"],
"size": other_fox_document["size"],
"created_at": other_fox_document["created_at"].isoformat(),
"updated_at": other_fox_document["updated_at"].isoformat(),
src/backend/core/views.py
+2 -11
View File
@@ -225,17 +225,8 @@ class DocumentView(views.APIView):
{"term": {enums.REACH: params.reach}}
)
# Filter by users and groups based on authentication$
# user = request.user
# groups = user.get_teams()
# search_body['query']['bool']['filter'].append({
# "terms": {"users": [user.sub]}
# })
# search_body['query']['bool']['filter'].append({
# "terms": {"groups": list(groups)}
# })
# Always filter out inactive documents
search_body["query"]["bool"]["filter"].append({"term": {"is_active": True}})
response = client.search(index=",".join(params.services), body=search_body)
return Response(response["hits"]["hits"], status=status.HTTP_200_OK)