Compare commits
32 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c6ed7a79f8 | |||
| f2106dd880 | |||
| 3652732852 | |||
| 5427f18ec0 | |||
| a83f140510 | |||
| 78e2bc690e | |||
| 4974b5cf31 | |||
| 337129cb48 | |||
| 33f1555377 | |||
| bc895512b4 | |||
| f9c0a2b3e5 | |||
| ad92bf614d | |||
| 57a6f73d9b | |||
| 9525b97286 | |||
| f49902a2b2 | |||
| ff08b3e977 | |||
| 1b36f6b729 | |||
| 47fd1dfd11 | |||
| 0145a0575f | |||
| 1a9011f5fa | |||
| cb77f6803b | |||
| 254a5418da | |||
| fd49f28449 | |||
| 82a0c1a770 | |||
| a758254b60 | |||
| 6314cb3a18 | |||
| 3e410e3519 | |||
| aba7959344 | |||
| 3d45c7c215 | |||
| cdb26b480a | |||
| 23a0f2761f | |||
| 0d596e338c |
@@ -43,6 +43,10 @@ venv.bak/
|
||||
env.d/development/*.local
|
||||
env.d/terraform
|
||||
|
||||
# Docker
|
||||
compose.override.yml
|
||||
docker/auth/*.local
|
||||
|
||||
# npm
|
||||
node_modules
|
||||
|
||||
|
||||
@@ -6,6 +6,21 @@ and this project adheres to
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- ✨(frontend) enable ODT export for documents #1524
|
||||
- ✨(frontend) improve mobile UX by showing subdocs count #1540
|
||||
|
||||
### Fixed
|
||||
|
||||
- ♿(frontend) improve accessibility:
|
||||
- ♿(frontend) improve ARIA in doc grid and editor for a11y #1519
|
||||
- ♿(frontend) improve accessibility and styling of summary table #1528
|
||||
- ♿(frontend) add focus trap and enter key support to remove doc modal #1531
|
||||
- 🐛(docx) fix image overflow by limiting width to 600px during export #1525
|
||||
- 🐛(frontend) preserve @ character when esc is pressed after typing it #1512
|
||||
- 🐛(frontend) fix pdf embed to use full width #1526
|
||||
|
||||
## [3.9.0] - 2025-11-10
|
||||
|
||||
### Added
|
||||
@@ -24,6 +39,7 @@ and this project adheres to
|
||||
|
||||
- 🐛(frontend) fix duplicate document entries in grid #1479
|
||||
- 🐛(backend) fix trashbin list #1520
|
||||
|
||||
- ♿(frontend) improve accessibility:
|
||||
- ♿(frontend) remove empty alt on logo due to Axe a11y error #1516
|
||||
- 🐛(backend) fix s3 version_id validation #1543
|
||||
@@ -81,6 +97,9 @@ and this project adheres to
|
||||
- ♿ update labels and shared document icon accessibility #1442
|
||||
- 🍱(frontend) Fonts GDPR compliants #1453
|
||||
- ♻️(service-worker) improve SW registration and update handling #1473
|
||||
- ✨(backend) add async indexation of documents on save (or access save) #1276
|
||||
- ✨(backend) add debounce mechanism to limit indexation jobs #1276
|
||||
- ✨(api) add API route to search for indexed documents in Find #1276
|
||||
|
||||
### Fixed
|
||||
|
||||
|
||||
@@ -213,7 +213,6 @@ logs: ## display app-dev logs (follow mode)
|
||||
.PHONY: logs
|
||||
|
||||
run-backend: ## Start only the backend application and all needed services
|
||||
@$(COMPOSE) up --force-recreate -d docspec
|
||||
@$(COMPOSE) up --force-recreate -d celery-dev
|
||||
@$(COMPOSE) up --force-recreate -d y-provider-development
|
||||
@$(COMPOSE) up --force-recreate -d nginx
|
||||
@@ -248,6 +247,10 @@ demo: ## flush db then create a demo for load testing purpose
|
||||
@$(MANAGE) create_demo
|
||||
.PHONY: demo
|
||||
|
||||
index: ## index all documents to remote search
|
||||
@$(MANAGE) index
|
||||
.PHONY: index
|
||||
|
||||
# Nota bene: Black should come after isort just in case they don't agree...
|
||||
lint: ## lint back-end python sources
|
||||
lint: \
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# shellcheck source=bin/_config.sh
|
||||
source "$(dirname "${BASH_SOURCE[0]}")/_config.sh"
|
||||
|
||||
_dc_run app-dev python -c 'from cryptography.fernet import Fernet;import sys; sys.stdout.write("\n" + Fernet.generate_key().decode() + "\n");'
|
||||
+17
-4
@@ -72,6 +72,11 @@ services:
|
||||
- env.d/development/postgresql.local
|
||||
ports:
|
||||
- "8071:8000"
|
||||
networks:
|
||||
default: {}
|
||||
lasuite:
|
||||
aliases:
|
||||
- impress
|
||||
volumes:
|
||||
- ./src/backend:/app
|
||||
- ./data/static:/data/static
|
||||
@@ -92,6 +97,9 @@ services:
|
||||
command: ["celery", "-A", "impress.celery_app", "worker", "-l", "DEBUG"]
|
||||
environment:
|
||||
- DJANGO_CONFIGURATION=Development
|
||||
networks:
|
||||
- default
|
||||
- lasuite
|
||||
env_file:
|
||||
- env.d/development/common
|
||||
- env.d/development/common.local
|
||||
@@ -107,6 +115,11 @@ services:
|
||||
image: nginx:1.25
|
||||
ports:
|
||||
- "8083:8083"
|
||||
networks:
|
||||
default: {}
|
||||
lasuite:
|
||||
aliases:
|
||||
- nginx
|
||||
volumes:
|
||||
- ./docker/files/etc/nginx/conf.d:/etc/nginx/conf.d:ro
|
||||
depends_on:
|
||||
@@ -218,7 +231,7 @@ services:
|
||||
condition: service_healthy
|
||||
restart: true
|
||||
|
||||
docspec:
|
||||
image: ghcr.io/docspecio/api:2.0.2
|
||||
ports:
|
||||
- "4000:4000"
|
||||
networks:
|
||||
lasuite:
|
||||
name: lasuite-network
|
||||
driver: bridge
|
||||
|
||||
@@ -12,6 +12,7 @@ flowchart TD
|
||||
Back --> DB("Database (PostgreSQL)")
|
||||
Back <--> Celery --> DB
|
||||
Back ----> S3("Minio (S3)")
|
||||
Back -- REST API --> Find
|
||||
```
|
||||
|
||||
### Architecture decision records
|
||||
|
||||
+7
-1
@@ -93,6 +93,13 @@ These are the environment variables you can set for the `impress-backend` contai
|
||||
| OIDC_USERINFO_SHORTNAME_FIELD | OIDC token claims to create shortname | first_name |
|
||||
| POSTHOG_KEY | Posthog key for analytics | |
|
||||
| REDIS_URL | Cache url | redis://redis:6379/1 |
|
||||
| SEARCH_INDEXER_CLASS | Class of the backend for document indexation & search | |
|
||||
| SEARCH_INDEXER_BATCH_SIZE | Size of each batch for indexation of all documents | 100000 |
|
||||
| SEARCH_INDEXER_COUNTDOWN | Minimum debounce delay of indexation jobs (in seconds) | 1 |
|
||||
| SEARCH_INDEXER_URL | Find application endpoint for indexation | |
|
||||
| SEARCH_INDEXER_SECRET | Token for indexation queries | |
|
||||
| SEARCH_INDEXER_QUERY_URL | Find application endpoint for search | |
|
||||
| SEARCH_INDEXER_QUERY_LIMIT | Maximum number of results expected from search endpoint | 50 |
|
||||
| SENTRY_DSN | Sentry host | |
|
||||
| SESSION_COOKIE_AGE | duration of the cookie session | 60*60*12 |
|
||||
| SPECTACULAR_SETTINGS_ENABLE_DJANGO_DEPLOY_CHECK | | false |
|
||||
@@ -103,7 +110,6 @@ These are the environment variables you can set for the `impress-backend` contai
|
||||
| USER_OIDC_ESSENTIAL_CLAIMS | Essential claims in OIDC token | [] |
|
||||
| Y_PROVIDER_API_BASE_URL | Y Provider url | |
|
||||
| Y_PROVIDER_API_KEY | Y provider API key | |
|
||||
| DOCSPEC_API_URL | URL to endpoint of DocSpec conversion API | |
|
||||
|
||||
|
||||
## impress-frontend image
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
# Setup the Find search for Impress
|
||||
|
||||
This configuration will enable the fulltext search feature for Docs :
|
||||
- Each save on **core.Document** or **core.DocumentAccess** will trigger the indexer
|
||||
- The `api/v1.0/documents/search/` will work as a proxy with the Find API for fulltext search.
|
||||
|
||||
## Create an index service for Docs
|
||||
|
||||
Configure a **Service** for Docs application with these settings
|
||||
|
||||
- **Name**: `docs`<br>_request.auth.name of the Docs application._
|
||||
- **Client id**: `impress`<br>_Name of the token audience or client_id of the Docs application._
|
||||
|
||||
See [how-to-use-indexer.md](how-to-use-indexer.md) for details.
|
||||
|
||||
## Configure settings of Docs
|
||||
|
||||
Add those Django settings the Docs application to enable the feature.
|
||||
|
||||
```shell
|
||||
SEARCH_INDEXER_CLASS="core.services.search_indexers.FindDocumentIndexer"
|
||||
SEARCH_INDEXER_COUNTDOWN=10 # Debounce delay in seconds for the indexer calls.
|
||||
|
||||
# The token from service "docs" of Find application (development).
|
||||
SEARCH_INDEXER_SECRET="find-api-key-for-docs-with-exactly-50-chars-length"
|
||||
SEARCH_INDEXER_URL="http://find:8000/api/v1.0/documents/index/"
|
||||
|
||||
# Search endpoint. Uses the OIDC token for authentication
|
||||
SEARCH_INDEXER_QUERY_URL="http://find:8000/api/v1.0/documents/search/"
|
||||
# Maximum number of results expected from the search endpoint
|
||||
SEARCH_INDEXER_QUERY_LIMIT=50
|
||||
```
|
||||
|
||||
We also need to enable the **OIDC Token** refresh or the authentication will fail quickly.
|
||||
|
||||
```shell
|
||||
# Store OIDC tokens in the session
|
||||
OIDC_STORE_ACCESS_TOKEN = True # Store the access token in the session
|
||||
OIDC_STORE_REFRESH_TOKEN = True # Store the encrypted refresh token in the session
|
||||
OIDC_STORE_REFRESH_TOKEN_KEY = "your-32-byte-encryption-key==" # Must be a valid Fernet key (32 url-safe base64-encoded bytes)
|
||||
```
|
||||
@@ -97,6 +97,17 @@ Production deployments differ significantly from development environments. The t
|
||||
| 5433 | PostgreSQL (Keycloak) |
|
||||
| 1081 | MailCatcher |
|
||||
|
||||
**With fulltext search service**
|
||||
|
||||
| Port | Service |
|
||||
| --------- | --------------------- |
|
||||
| 8081 | Find (Django) |
|
||||
| 9200 | Opensearch |
|
||||
| 9600 | Opensearch admin |
|
||||
| 5601 | Opensearch dashboard |
|
||||
| 25432 | PostgreSQL (Find) |
|
||||
|
||||
|
||||
## 6. Sizing Guidelines
|
||||
|
||||
**RAM** – start at 8 GB dev / 16 GB staging / 32 GB prod. Postgres and Keycloak are the first to OOM; scale them first.
|
||||
|
||||
@@ -36,6 +36,7 @@ OIDC_OP_JWKS_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/c
|
||||
OIDC_OP_AUTHORIZATION_ENDPOINT=http://localhost:8083/realms/impress/protocol/openid-connect/auth
|
||||
OIDC_OP_TOKEN_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token
|
||||
OIDC_OP_USER_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/userinfo
|
||||
OIDC_OP_INTROSPECTION_ENDPOINT=http://nginx:8083/realms/impress/protocol/openid-connect/token/introspect
|
||||
|
||||
OIDC_RP_CLIENT_ID=impress
|
||||
OIDC_RP_CLIENT_SECRET=ThisIsAnExampleKeyForDevPurposeOnly
|
||||
@@ -49,6 +50,14 @@ LOGOUT_REDIRECT_URL=http://localhost:3000
|
||||
OIDC_REDIRECT_ALLOWED_HOSTS=["http://localhost:8083", "http://localhost:3000"]
|
||||
OIDC_AUTH_REQUEST_EXTRA_PARAMS={"acr_values": "eidas1"}
|
||||
|
||||
# Store OIDC tokens in the session
|
||||
OIDC_STORE_ACCESS_TOKEN = True
|
||||
OIDC_STORE_REFRESH_TOKEN = True # Store the encrypted refresh token in the session.
|
||||
|
||||
# Must be a valid Fernet key (32 url-safe base64-encoded bytes)
|
||||
# To create one, use the bin/fernetkey command.
|
||||
# OIDC_STORE_REFRESH_TOKEN_KEY="your-32-byte-encryption-key=="
|
||||
|
||||
# AI
|
||||
AI_FEATURE_ENABLED=true
|
||||
AI_BASE_URL=https://openaiendpoint.com
|
||||
@@ -67,7 +76,11 @@ DJANGO_SERVER_TO_SERVER_API_TOKENS=server-api-token
|
||||
Y_PROVIDER_API_BASE_URL=http://y-provider-development:4444/api/
|
||||
Y_PROVIDER_API_KEY=yprovider-api-key
|
||||
|
||||
DOCSPEC_API_URL=http://docspec:4000/conversion
|
||||
|
||||
# Theme customization
|
||||
THEME_CUSTOMIZATION_CACHE_TIMEOUT=15
|
||||
THEME_CUSTOMIZATION_CACHE_TIMEOUT=15
|
||||
|
||||
# Indexer (disabled)
|
||||
# SEARCH_INDEXER_CLASS="core.services.search_indexers.SearchIndexer"
|
||||
SEARCH_INDEXER_SECRET=find-api-key-for-docs-with-exactly-50-chars-length # Key generated by create_demo in Find app.
|
||||
SEARCH_INDEXER_URL="http://find:8000/api/v1.0/documents/index/"
|
||||
SEARCH_INDEXER_QUERY_URL="http://find:8000/api/v1.0/documents/search/"
|
||||
|
||||
@@ -3,7 +3,6 @@ BURST_THROTTLE_RATES="200/minute"
|
||||
COLLABORATION_API_URL=http://y-provider:4444/collaboration/api/
|
||||
SUSTAINED_THROTTLE_RATES="200/hour"
|
||||
Y_PROVIDER_API_BASE_URL=http://y-provider:4444/api/
|
||||
DOCSPEC_API_URL=http://docspec:4000/conversion
|
||||
|
||||
# Throttle
|
||||
API_DOCUMENT_THROTTLE_RATE=1000/min
|
||||
|
||||
@@ -10,7 +10,6 @@ from django.utils.functional import lazy
|
||||
from django.utils.text import slugify
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from core.services import mime_types
|
||||
import magic
|
||||
from rest_framework import serializers
|
||||
|
||||
@@ -18,7 +17,7 @@ from core import choices, enums, models, utils, validators
|
||||
from core.services.ai_services import AI_ACTIONS
|
||||
from core.services.converter_services import (
|
||||
ConversionError,
|
||||
Converter,
|
||||
YdocConverter,
|
||||
)
|
||||
|
||||
|
||||
@@ -188,7 +187,6 @@ class DocumentSerializer(ListDocumentSerializer):
|
||||
|
||||
content = serializers.CharField(required=False)
|
||||
websocket = serializers.BooleanField(required=False, write_only=True)
|
||||
file = serializers.FileField(required=False, write_only=True, allow_null=True)
|
||||
|
||||
class Meta:
|
||||
model = models.Document
|
||||
@@ -205,7 +203,6 @@ class DocumentSerializer(ListDocumentSerializer):
|
||||
"deleted_at",
|
||||
"depth",
|
||||
"excerpt",
|
||||
"file",
|
||||
"is_favorite",
|
||||
"link_role",
|
||||
"link_reach",
|
||||
@@ -463,11 +460,7 @@ class ServerCreateDocumentSerializer(serializers.Serializer):
|
||||
language = user.language or language
|
||||
|
||||
try:
|
||||
document_content = Converter().convert(
|
||||
validated_data["content"],
|
||||
mime_types.MARKDOWN,
|
||||
mime_types.YJS
|
||||
)
|
||||
document_content = YdocConverter().convert(validated_data["content"])
|
||||
except ConversionError as err:
|
||||
raise serializers.ValidationError(
|
||||
{"content": ["Could not convert content"]}
|
||||
@@ -896,3 +889,13 @@ class MoveDocumentSerializer(serializers.Serializer):
|
||||
choices=enums.MoveNodePositionChoices.choices,
|
||||
default=enums.MoveNodePositionChoices.LAST_CHILD,
|
||||
)
|
||||
|
||||
|
||||
class SearchDocumentSerializer(serializers.Serializer):
|
||||
"""Serializer for fulltext search requests through Find application"""
|
||||
|
||||
q = serializers.CharField(required=True, allow_blank=False, trim_whitespace=True)
|
||||
page_size = serializers.IntegerField(
|
||||
required=False, min_value=1, max_value=50, default=20
|
||||
)
|
||||
page = serializers.IntegerField(required=False, min_value=1, default=1)
|
||||
|
||||
@@ -21,6 +21,7 @@ from django.db.models.expressions import RawSQL
|
||||
from django.db.models.functions import Left, Length
|
||||
from django.http import Http404, StreamingHttpResponse
|
||||
from django.urls import reverse
|
||||
from django.utils.decorators import method_decorator
|
||||
from django.utils.functional import cached_property
|
||||
from django.utils.text import capfirst, slugify
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
@@ -31,6 +32,7 @@ from botocore.exceptions import ClientError
|
||||
from csp.constants import NONE
|
||||
from csp.decorators import csp_update
|
||||
from lasuite.malware_detection import malware_detection
|
||||
from lasuite.oidc_login.decorators import refresh_oidc_access_token
|
||||
from rest_framework import filters, status, viewsets
|
||||
from rest_framework import response as drf_response
|
||||
from rest_framework.permissions import AllowAny
|
||||
@@ -39,12 +41,18 @@ from core import authentication, choices, enums, models
|
||||
from core.services.ai_services import AIService
|
||||
from core.services.collaboration_services import CollaborationService
|
||||
from core.services.converter_services import (
|
||||
ConversionError,
|
||||
ServiceUnavailableError as YProviderServiceUnavailableError,
|
||||
ValidationError as YProviderValidationError,
|
||||
Converter,
|
||||
)
|
||||
from core.services import mime_types
|
||||
from core.services.converter_services import (
|
||||
ValidationError as YProviderValidationError,
|
||||
)
|
||||
from core.services.converter_services import (
|
||||
YdocConverter,
|
||||
)
|
||||
from core.services.search_indexers import (
|
||||
get_document_indexer,
|
||||
get_visited_document_ids_of,
|
||||
)
|
||||
from core.tasks.mail import send_ask_for_access_mail
|
||||
from core.utils import extract_attachments, filter_descendants
|
||||
|
||||
@@ -371,6 +379,7 @@ class DocumentViewSet(
|
||||
list_serializer_class = serializers.ListDocumentSerializer
|
||||
trashbin_serializer_class = serializers.ListDocumentSerializer
|
||||
tree_serializer_class = serializers.ListDocumentSerializer
|
||||
search_serializer_class = serializers.ListDocumentSerializer
|
||||
|
||||
def get_queryset(self):
|
||||
"""Get queryset performing all annotation and filtering on the document tree structure."""
|
||||
@@ -501,27 +510,6 @@ class DocumentViewSet(
|
||||
"IN SHARE ROW EXCLUSIVE MODE;"
|
||||
)
|
||||
|
||||
# Remove file from validated_data as it's not a model field
|
||||
# Process it if present
|
||||
uploaded_file = serializer.validated_data.pop("file", None)
|
||||
|
||||
# If a file is uploaded, convert it to Yjs format and set as content
|
||||
if uploaded_file:
|
||||
try:
|
||||
file_content = uploaded_file.read()
|
||||
|
||||
converter = Converter()
|
||||
converted_content = converter.convert(
|
||||
file_content,
|
||||
content_type=uploaded_file.content_type,
|
||||
accept=mime_types.YJS
|
||||
)
|
||||
serializer.validated_data["content"] = converted_content
|
||||
except ConversionError as err:
|
||||
raise drf.exceptions.ValidationError(
|
||||
{"file": ["Could not convert file content"]}
|
||||
) from err
|
||||
|
||||
obj = models.Document.add_root(
|
||||
creator=self.request.user,
|
||||
**serializer.validated_data,
|
||||
@@ -1083,6 +1071,83 @@ class DocumentViewSet(
|
||||
{"id": str(duplicated_document.id)}, status=status.HTTP_201_CREATED
|
||||
)
|
||||
|
||||
def _search_simple(self, request, text):
|
||||
"""
|
||||
Returns a queryset filtered by the content of the document title
|
||||
"""
|
||||
# As the 'list' view we get a prefiltered queryset (deleted docs are excluded)
|
||||
queryset = self.get_queryset()
|
||||
filterset = DocumentFilter({"title": text}, queryset=queryset)
|
||||
|
||||
if not filterset.is_valid():
|
||||
raise drf.exceptions.ValidationError(filterset.errors)
|
||||
|
||||
queryset = filterset.filter_queryset(queryset)
|
||||
|
||||
return self.get_response_for_queryset(
|
||||
queryset.order_by("-updated_at"),
|
||||
context={
|
||||
"request": request,
|
||||
},
|
||||
)
|
||||
|
||||
def _search_fulltext(self, indexer, request, params):
|
||||
"""
|
||||
Returns a queryset from the results the fulltext search of Find
|
||||
"""
|
||||
access_token = request.session.get("oidc_access_token")
|
||||
user = request.user
|
||||
text = params.validated_data["q"]
|
||||
queryset = models.Document.objects.all()
|
||||
|
||||
# Retrieve the documents ids from Find.
|
||||
results = indexer.search(
|
||||
text=text,
|
||||
token=access_token,
|
||||
visited=get_visited_document_ids_of(queryset, user),
|
||||
)
|
||||
|
||||
docs_by_uuid = {str(d.pk): d for d in queryset.filter(pk__in=results)}
|
||||
ordered_docs = [docs_by_uuid[id] for id in results]
|
||||
|
||||
page = self.paginate_queryset(ordered_docs)
|
||||
|
||||
serializer = self.get_serializer(
|
||||
page if page else ordered_docs,
|
||||
many=True,
|
||||
context={
|
||||
"request": request,
|
||||
},
|
||||
)
|
||||
|
||||
return self.get_paginated_response(serializer.data)
|
||||
|
||||
@drf.decorators.action(detail=False, methods=["get"], url_path="search")
|
||||
@method_decorator(refresh_oidc_access_token)
|
||||
def search(self, request, *args, **kwargs):
|
||||
"""
|
||||
Returns a DRF response containing the filtered, annotated and ordered document list.
|
||||
|
||||
Applies filtering based on request parameter 'q' from `SearchDocumentSerializer`.
|
||||
Depending of the configuration it can be:
|
||||
- A fulltext search through the opensearch indexation app "find" if the backend is
|
||||
enabled (see SEARCH_INDEXER_CLASS)
|
||||
- A filtering by the model field 'title'.
|
||||
|
||||
The ordering is always by the most recent first.
|
||||
"""
|
||||
params = serializers.SearchDocumentSerializer(data=request.query_params)
|
||||
params.is_valid(raise_exception=True)
|
||||
|
||||
indexer = get_document_indexer()
|
||||
|
||||
if indexer:
|
||||
return self._search_fulltext(indexer, request, params=params)
|
||||
|
||||
# The indexer is not configured, we fallback on a simple icontains filter by the
|
||||
# model field 'title'.
|
||||
return self._search_simple(request, text=params.validated_data["q"])
|
||||
|
||||
@drf.decorators.action(detail=True, methods=["get"], url_path="versions")
|
||||
def versions_list(self, request, *args, **kwargs):
|
||||
"""
|
||||
@@ -1621,14 +1686,14 @@ class DocumentViewSet(
|
||||
if base64_content is not None:
|
||||
# Convert using the y-provider service
|
||||
try:
|
||||
yprovider = Converter()
|
||||
yprovider = YdocConverter()
|
||||
result = yprovider.convert(
|
||||
base64.b64decode(base64_content),
|
||||
mime_types.YJS,
|
||||
"application/vnd.yjs.doc",
|
||||
{
|
||||
"markdown": mime_types.MARKDOWN,
|
||||
"html": mime_types.HTML,
|
||||
"json": mime_types.JSON,
|
||||
"markdown": "text/markdown",
|
||||
"html": "text/html",
|
||||
"json": "application/json",
|
||||
}[content_format],
|
||||
)
|
||||
content = result
|
||||
|
||||
@@ -1,11 +1,19 @@
|
||||
"""Impress Core application"""
|
||||
# from django.apps import AppConfig
|
||||
# from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from django.apps import AppConfig
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
|
||||
# class CoreConfig(AppConfig):
|
||||
# """Configuration class for the impress core app."""
|
||||
class CoreConfig(AppConfig):
|
||||
"""Configuration class for the impress core app."""
|
||||
|
||||
# name = "core"
|
||||
# app_label = "core"
|
||||
# verbose_name = _("impress core application")
|
||||
name = "core"
|
||||
app_label = "core"
|
||||
verbose_name = _("Impress core application")
|
||||
|
||||
def ready(self):
|
||||
"""
|
||||
Import signals when the app is ready.
|
||||
"""
|
||||
# pylint: disable=import-outside-toplevel, unused-import
|
||||
from . import signals # noqa: PLC0415
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""
|
||||
Handle search setup that needs to be done at bootstrap time.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
from core.services.search_indexers import get_document_indexer
|
||||
|
||||
logger = logging.getLogger("docs.search.bootstrap_search")
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""Index all documents to remote search service"""
|
||||
|
||||
help = __doc__
|
||||
|
||||
def add_arguments(self, parser):
|
||||
"""Add argument to require forcing execution when not in debug mode."""
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
action="store",
|
||||
dest="batch_size",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Indexation query batch size",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
"""Launch and log search index generation."""
|
||||
indexer = get_document_indexer()
|
||||
|
||||
if not indexer:
|
||||
raise CommandError("The indexer is not enabled or properly configured.")
|
||||
|
||||
logger.info("Starting to regenerate Find index...")
|
||||
start = time.perf_counter()
|
||||
batch_size = options["batch_size"]
|
||||
|
||||
try:
|
||||
count = indexer.index(batch_size=batch_size)
|
||||
except Exception as err:
|
||||
raise CommandError("Unable to regenerate index") from err
|
||||
|
||||
duration = time.perf_counter() - start
|
||||
logger.info(
|
||||
"Search index regenerated from %d document(s) in %.2f seconds.",
|
||||
count,
|
||||
duration,
|
||||
)
|
||||
+27
-23
@@ -432,32 +432,35 @@ class Document(MP_Node, BaseModel):
|
||||
def save(self, *args, **kwargs):
|
||||
"""Write content to object storage only if _content has changed."""
|
||||
super().save(*args, **kwargs)
|
||||
|
||||
if self._content:
|
||||
file_key = self.file_key
|
||||
bytes_content = self._content.encode("utf-8")
|
||||
self.save_content(self._content)
|
||||
|
||||
# Attempt to directly check if the object exists using the storage client.
|
||||
try:
|
||||
response = default_storage.connection.meta.client.head_object(
|
||||
Bucket=default_storage.bucket_name, Key=file_key
|
||||
)
|
||||
except ClientError as excpt:
|
||||
# If the error is a 404, the object doesn't exist, so we should create it.
|
||||
if excpt.response["Error"]["Code"] == "404":
|
||||
has_changed = True
|
||||
else:
|
||||
raise
|
||||
def save_content(self, content):
|
||||
"""Save content to object storage."""
|
||||
|
||||
file_key = self.file_key
|
||||
bytes_content = content.encode("utf-8")
|
||||
|
||||
# Attempt to directly check if the object exists using the storage client.
|
||||
try:
|
||||
response = default_storage.connection.meta.client.head_object(
|
||||
Bucket=default_storage.bucket_name, Key=file_key
|
||||
)
|
||||
except ClientError as excpt:
|
||||
# If the error is a 404, the object doesn't exist, so we should create it.
|
||||
if excpt.response["Error"]["Code"] == "404":
|
||||
has_changed = True
|
||||
else:
|
||||
# Compare the existing ETag with the MD5 hash of the new content.
|
||||
has_changed = (
|
||||
response["ETag"].strip('"')
|
||||
!= hashlib.md5(bytes_content).hexdigest() # noqa: S324
|
||||
)
|
||||
raise
|
||||
else:
|
||||
# Compare the existing ETag with the MD5 hash of the new content.
|
||||
has_changed = (
|
||||
response["ETag"].strip('"') != hashlib.md5(bytes_content).hexdigest() # noqa: S324
|
||||
)
|
||||
|
||||
if has_changed:
|
||||
content_file = ContentFile(bytes_content)
|
||||
default_storage.save(file_key, content_file)
|
||||
if has_changed:
|
||||
content_file = ContentFile(bytes_content)
|
||||
default_storage.save(file_key, content_file)
|
||||
|
||||
def is_leaf(self):
|
||||
"""
|
||||
@@ -901,7 +904,8 @@ class Document(MP_Node, BaseModel):
|
||||
|
||||
# Mark all descendants as soft deleted
|
||||
self.get_descendants().filter(ancestors_deleted_at__isnull=True).update(
|
||||
ancestors_deleted_at=self.ancestors_deleted_at
|
||||
ancestors_deleted_at=self.ancestors_deleted_at,
|
||||
updated_at=self.updated_at,
|
||||
)
|
||||
|
||||
@transaction.atomic
|
||||
|
||||
@@ -5,9 +5,7 @@ from base64 import b64encode
|
||||
from django.conf import settings
|
||||
|
||||
import requests
|
||||
import typing
|
||||
|
||||
from core.services import mime_types
|
||||
|
||||
class ConversionError(Exception):
|
||||
"""Base exception for conversion-related errors."""
|
||||
@@ -21,65 +19,8 @@ class ServiceUnavailableError(ConversionError):
|
||||
"""Raised when the conversion service is unavailable."""
|
||||
|
||||
|
||||
class ConverterProtocol(typing.Protocol):
|
||||
def convert(self, text, content_type, accept): ...
|
||||
|
||||
|
||||
class Converter:
|
||||
docspec: ConverterProtocol
|
||||
ydoc: ConverterProtocol
|
||||
|
||||
def __init__(self):
|
||||
self.docspec = DocSpecConverter()
|
||||
self.ydoc = YdocConverter()
|
||||
|
||||
def convert(self, input, content_type, accept):
|
||||
"""Convert input into other formats using external microservices."""
|
||||
|
||||
if content_type == mime_types.DOCX and accept == mime_types.YJS:
|
||||
return self.convert(
|
||||
self.docspec.convert(input, mime_types.DOCX, mime_types.BLOCKNOTE),
|
||||
mime_types.BLOCKNOTE,
|
||||
mime_types.YJS
|
||||
)
|
||||
|
||||
return self.ydoc.convert(input, content_type, accept)
|
||||
|
||||
|
||||
class DocSpecConverter:
|
||||
"""Service class for DocSpec conversion-related operations."""
|
||||
|
||||
def _request(self, url, data, content_type):
|
||||
"""Make a request to the DocSpec API."""
|
||||
|
||||
response = requests.post(
|
||||
url,
|
||||
headers={"Accept": mime_types.BLOCKNOTE},
|
||||
files={"file": ("document.docx", data, content_type)},
|
||||
timeout=settings.CONVERSION_API_TIMEOUT,
|
||||
verify=settings.CONVERSION_API_SECURE,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
|
||||
def convert(self, data, content_type, accept):
|
||||
"""Convert a Document to BlockNote."""
|
||||
if not data:
|
||||
raise ValidationError("Input data cannot be empty")
|
||||
|
||||
if content_type != mime_types.DOCX or accept != mime_types.BLOCKNOTE:
|
||||
raise ValidationError(f"Conversion from {content_type} to {accept} is not supported.")
|
||||
|
||||
try:
|
||||
return self._request(settings.DOCSPEC_API_URL, data, content_type).content
|
||||
except requests.RequestException as err:
|
||||
raise ServiceUnavailableError(
|
||||
"Failed to connect to DocSpec conversion service",
|
||||
) from err
|
||||
|
||||
|
||||
class YdocConverter:
|
||||
"""Service class for YDoc conversion-related operations."""
|
||||
"""Service class for conversion-related operations."""
|
||||
|
||||
@property
|
||||
def auth_header(self):
|
||||
@@ -104,7 +45,7 @@ class YdocConverter:
|
||||
return response
|
||||
|
||||
def convert(
|
||||
self, text, content_type=mime_types.MARKDOWN, accept=mime_types.YJS
|
||||
self, text, content_type="text/markdown", accept="application/vnd.yjs.doc"
|
||||
):
|
||||
"""Convert a Markdown text into our internal format using an external microservice."""
|
||||
|
||||
@@ -118,14 +59,14 @@ class YdocConverter:
|
||||
content_type,
|
||||
accept,
|
||||
)
|
||||
if accept == mime_types.YJS:
|
||||
if accept == "application/vnd.yjs.doc":
|
||||
return b64encode(response.content).decode("utf-8")
|
||||
if accept in {mime_types.MARKDOWN, "text/html"}:
|
||||
if accept in {"text/markdown", "text/html"}:
|
||||
return response.text
|
||||
if accept == mime_types.JSON:
|
||||
if accept == "application/json":
|
||||
return response.json()
|
||||
raise ValidationError("Unsupported format")
|
||||
except requests.RequestException as err:
|
||||
raise ServiceUnavailableError(
|
||||
f"Failed to connect to YDoc conversion service {content_type}, {accept}",
|
||||
"Failed to connect to conversion service",
|
||||
) from err
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
BLOCKNOTE = "application/vnd.blocknote+json"
|
||||
YJS = "application/vnd.yjs.doc"
|
||||
MARKDOWN = "text/markdown"
|
||||
JSON = "application/json"
|
||||
DOCX = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
|
||||
HTML = "text/html"
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Document search index management utilities and indexers"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from functools import cache
|
||||
|
||||
from django.conf import settings
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
from django.db.models import Subquery
|
||||
from django.utils.module_loading import import_string
|
||||
|
||||
import requests
|
||||
|
||||
from core import models, utils
|
||||
from core.services.converter_services import (
|
||||
ServiceUnavailableError as YProviderServiceUnavailableError,
|
||||
)
|
||||
from core.services.converter_services import (
|
||||
YdocConverter,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@cache
|
||||
def get_document_indexer():
|
||||
"""Returns an instance of indexer service if enabled and properly configured."""
|
||||
classpath = settings.SEARCH_INDEXER_CLASS
|
||||
|
||||
# For this usecase an empty indexer class is not an issue but a feature.
|
||||
if not classpath:
|
||||
logger.info("Document indexer is not configured (see SEARCH_INDEXER_CLASS)")
|
||||
return None
|
||||
|
||||
try:
|
||||
indexer_class = import_string(settings.SEARCH_INDEXER_CLASS)
|
||||
return indexer_class()
|
||||
except ImportError as err:
|
||||
logger.error("SEARCH_INDEXER_CLASS setting is not valid : %s", err)
|
||||
except ImproperlyConfigured as err:
|
||||
logger.error("Document indexer is not properly configured : %s", err)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_batch_accesses_by_users_and_teams(paths):
|
||||
"""
|
||||
Get accesses related to a list of document paths,
|
||||
grouped by users and teams, including all ancestor paths.
|
||||
"""
|
||||
ancestor_map = utils.get_ancestor_to_descendants_map(
|
||||
paths, steplen=models.Document.steplen
|
||||
)
|
||||
ancestor_paths = list(ancestor_map.keys())
|
||||
|
||||
access_qs = models.DocumentAccess.objects.filter(
|
||||
document__path__in=ancestor_paths
|
||||
).values("document__path", "user__sub", "team")
|
||||
|
||||
access_by_document_path = defaultdict(lambda: {"users": set(), "teams": set()})
|
||||
|
||||
for access in access_qs:
|
||||
ancestor_path = access["document__path"]
|
||||
user_sub = access["user__sub"]
|
||||
team = access["team"]
|
||||
|
||||
for descendant_path in ancestor_map.get(ancestor_path, []):
|
||||
if user_sub:
|
||||
access_by_document_path[descendant_path]["users"].add(str(user_sub))
|
||||
if team:
|
||||
access_by_document_path[descendant_path]["teams"].add(team)
|
||||
|
||||
return dict(access_by_document_path)
|
||||
|
||||
|
||||
def get_visited_document_ids_of(queryset, user):
|
||||
"""
|
||||
Returns the ids of the documents that have a linktrace to the user and NOT owned.
|
||||
It will be use to limit the opensearch responses to the public documents already
|
||||
"visited" by the user.
|
||||
"""
|
||||
if isinstance(user, AnonymousUser):
|
||||
return []
|
||||
|
||||
qs = models.LinkTrace.objects.filter(user=user)
|
||||
|
||||
docs = (
|
||||
queryset.exclude(accesses__user=user)
|
||||
.filter(
|
||||
deleted_at__isnull=True,
|
||||
ancestors_deleted_at__isnull=True,
|
||||
)
|
||||
.filter(pk__in=Subquery(qs.values("document_id")))
|
||||
.order_by("pk")
|
||||
.distinct("pk")
|
||||
)
|
||||
|
||||
return [str(id) for id in docs.values_list("pk", flat=True)]
|
||||
|
||||
|
||||
class BaseDocumentIndexer(ABC):
|
||||
"""
|
||||
Base class for document indexers.
|
||||
|
||||
Handles batching and access resolution. Subclasses must implement both
|
||||
`serialize_document()` and `push()` to define backend-specific behavior.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
Initialize the indexer.
|
||||
"""
|
||||
self.batch_size = settings.SEARCH_INDEXER_BATCH_SIZE
|
||||
self.indexer_url = settings.SEARCH_INDEXER_URL
|
||||
self.indexer_secret = settings.SEARCH_INDEXER_SECRET
|
||||
self.search_url = settings.SEARCH_INDEXER_QUERY_URL
|
||||
self.search_limit = settings.SEARCH_INDEXER_QUERY_LIMIT
|
||||
|
||||
if not self.indexer_url:
|
||||
raise ImproperlyConfigured(
|
||||
"SEARCH_INDEXER_URL must be set in Django settings."
|
||||
)
|
||||
|
||||
if not self.indexer_secret:
|
||||
raise ImproperlyConfigured(
|
||||
"SEARCH_INDEXER_SECRET must be set in Django settings."
|
||||
)
|
||||
|
||||
if not self.search_url:
|
||||
raise ImproperlyConfigured(
|
||||
"SEARCH_INDEXER_QUERY_URL must be set in Django settings."
|
||||
)
|
||||
|
||||
def index(self, queryset=None, batch_size=None):
|
||||
"""
|
||||
Fetch documents in batches, serialize them, and push to the search backend.
|
||||
|
||||
Args:
|
||||
queryset (optional): Document queryset
|
||||
Defaults to all documents without filter.
|
||||
batch_size (int, optional): Number of documents per batch.
|
||||
Defaults to settings.SEARCH_INDEXER_BATCH_SIZE.
|
||||
"""
|
||||
last_id = 0
|
||||
count = 0
|
||||
queryset = queryset or models.Document.objects.all()
|
||||
batch_size = batch_size or self.batch_size
|
||||
|
||||
while True:
|
||||
documents_batch = list(
|
||||
queryset.filter(
|
||||
id__gt=last_id,
|
||||
).order_by("id")[:batch_size]
|
||||
)
|
||||
|
||||
if not documents_batch:
|
||||
break
|
||||
|
||||
doc_paths = [doc.path for doc in documents_batch]
|
||||
last_id = documents_batch[-1].id
|
||||
accesses_by_document_path = get_batch_accesses_by_users_and_teams(doc_paths)
|
||||
|
||||
serialized_batch = [
|
||||
self.serialize_document(document, accesses_by_document_path)
|
||||
for document in documents_batch
|
||||
if document.content or document.title
|
||||
]
|
||||
|
||||
if serialized_batch:
|
||||
self.push(serialized_batch)
|
||||
count += len(serialized_batch)
|
||||
|
||||
return count
|
||||
|
||||
@abstractmethod
|
||||
def serialize_document(self, document, accesses):
|
||||
"""
|
||||
Convert a Document instance to a JSON-serializable format for indexing.
|
||||
|
||||
Must be implemented by subclasses.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def push(self, data):
|
||||
"""
|
||||
Push a batch of serialized documents to the backend.
|
||||
|
||||
Must be implemented by subclasses.
|
||||
"""
|
||||
|
||||
# pylint: disable-next=too-many-arguments,too-many-positional-arguments
|
||||
def search(self, text, token, visited=(), nb_results=None):
|
||||
"""
|
||||
Search for documents in Find app.
|
||||
Ensure the same default ordering as "Docs" list : -updated_at
|
||||
|
||||
Returns ids of the documents
|
||||
|
||||
Args:
|
||||
text (str): Text search content.
|
||||
token (str): OIDC Authentication token.
|
||||
visited (list, optional):
|
||||
List of ids of active public documents with LinkTrace
|
||||
Defaults to settings.SEARCH_INDEXER_BATCH_SIZE.
|
||||
nb_results (int, optional):
|
||||
The number of results to return.
|
||||
Defaults to 50 if not specified.
|
||||
"""
|
||||
nb_results = nb_results or self.search_limit
|
||||
response = self.search_query(
|
||||
data={
|
||||
"q": text,
|
||||
"visited": visited,
|
||||
"services": ["docs"],
|
||||
"nb_results": nb_results,
|
||||
"order_by": "updated_at",
|
||||
"order_direction": "desc",
|
||||
},
|
||||
token=token,
|
||||
)
|
||||
|
||||
return [d["_id"] for d in response]
|
||||
|
||||
@abstractmethod
|
||||
def search_query(self, data, token) -> dict:
|
||||
"""
|
||||
Retrieve documents from the Find app API.
|
||||
|
||||
Must be implemented by subclasses.
|
||||
"""
|
||||
|
||||
|
||||
class SearchIndexer(BaseDocumentIndexer):
|
||||
"""
|
||||
Document indexer that pushes documents to La Suite Find app.
|
||||
"""
|
||||
|
||||
def to_markdown(self, document):
|
||||
"""
|
||||
Convert document as markdown.
|
||||
Returns raw text if Ydoc service is not accessible
|
||||
"""
|
||||
content = ""
|
||||
base64_content = document.content
|
||||
|
||||
if base64_content is not None and len(base64_content) > 0:
|
||||
# Convert using the y-provider service
|
||||
try:
|
||||
yprovider = YdocConverter()
|
||||
result = yprovider.convert(
|
||||
base64.b64decode(base64_content),
|
||||
"application/vnd.yjs.doc",
|
||||
"text/markdown",
|
||||
)
|
||||
content = result
|
||||
except YProviderServiceUnavailableError as e:
|
||||
logger.error(
|
||||
"Error getting content for document %s: %s", document.pk, e
|
||||
)
|
||||
return utils.base64_yjs_to_text(base64_content)
|
||||
|
||||
return content
|
||||
|
||||
def serialize_document(self, document, accesses):
|
||||
"""
|
||||
Convert a Document to the JSON format expected by La Suite Find.
|
||||
|
||||
Args:
|
||||
document (Document): The document instance.
|
||||
accesses (dict): Mapping of document ID to user/team access.
|
||||
|
||||
Returns:
|
||||
dict: A JSON-serializable dictionary.
|
||||
"""
|
||||
doc_path = document.path
|
||||
text_content = self.to_markdown(document)
|
||||
|
||||
return {
|
||||
"id": str(document.id),
|
||||
"title": document.title or "",
|
||||
"content": text_content,
|
||||
"depth": document.depth,
|
||||
"path": document.path,
|
||||
"numchild": document.numchild,
|
||||
"created_at": document.created_at.isoformat(),
|
||||
"updated_at": document.updated_at.isoformat(),
|
||||
"users": list(accesses.get(doc_path, {}).get("users", set())),
|
||||
"groups": list(accesses.get(doc_path, {}).get("teams", set())),
|
||||
"reach": document.computed_link_reach,
|
||||
"size": len(text_content.encode("utf-8")),
|
||||
"mimetype": "text/markdown",
|
||||
"is_active": not bool(document.ancestors_deleted_at),
|
||||
}
|
||||
|
||||
def search_query(self, data, token) -> requests.Response:
|
||||
"""
|
||||
Retrieve documents from the Find app API.
|
||||
|
||||
Args:
|
||||
data (dict): search data
|
||||
token (str): OICD token
|
||||
|
||||
Returns:
|
||||
dict: A JSON-serializable dictionary.
|
||||
"""
|
||||
response = requests.post(
|
||||
self.search_url,
|
||||
json=data,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
timeout=10,
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
|
||||
def push(self, data):
|
||||
"""
|
||||
Push a batch of documents to the Find backend.
|
||||
|
||||
Args:
|
||||
data (list): List of document dictionaries.
|
||||
"""
|
||||
response = requests.post(
|
||||
self.indexer_url,
|
||||
json=data,
|
||||
headers={"Authorization": f"Bearer {self.indexer_secret}"},
|
||||
timeout=10,
|
||||
)
|
||||
response.raise_for_status()
|
||||
@@ -0,0 +1,33 @@
|
||||
"""
|
||||
Declare and configure the signals for the impress core application
|
||||
"""
|
||||
|
||||
from functools import partial
|
||||
|
||||
from django.db import transaction
|
||||
from django.db.models import signals
|
||||
from django.dispatch import receiver
|
||||
|
||||
from . import models
|
||||
from .tasks.search import trigger_batch_document_indexer
|
||||
|
||||
|
||||
@receiver(signals.post_save, sender=models.Document)
|
||||
def document_post_save(sender, instance, **kwargs): # pylint: disable=unused-argument
|
||||
"""
|
||||
Asynchronous call to the document indexer at the end of the transaction.
|
||||
Note : Within the transaction we can have an empty content and a serialization
|
||||
error.
|
||||
"""
|
||||
transaction.on_commit(partial(trigger_batch_document_indexer, instance))
|
||||
|
||||
|
||||
@receiver(signals.post_save, sender=models.DocumentAccess)
|
||||
def document_access_post_save(sender, instance, created, **kwargs): # pylint: disable=unused-argument
|
||||
"""
|
||||
Asynchronous call to the document indexer at the end of the transaction.
|
||||
"""
|
||||
if not created:
|
||||
transaction.on_commit(
|
||||
partial(trigger_batch_document_indexer, instance.document)
|
||||
)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Trigger document indexation using celery task."""
|
||||
|
||||
from logging import getLogger
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.cache import cache
|
||||
from django.db.models import Q
|
||||
|
||||
from django_redis.cache import RedisCache
|
||||
|
||||
from core import models
|
||||
from core.services.search_indexers import (
|
||||
get_document_indexer,
|
||||
)
|
||||
|
||||
from impress.celery_app import app
|
||||
|
||||
logger = getLogger(__file__)
|
||||
|
||||
|
||||
@app.task
|
||||
def document_indexer_task(document_id):
|
||||
"""Celery Task : Sends indexation query for a document."""
|
||||
indexer = get_document_indexer()
|
||||
|
||||
if indexer:
|
||||
logger.info("Start document %s indexation", document_id)
|
||||
indexer.index(models.Document.objects.filter(pk=document_id))
|
||||
|
||||
|
||||
def batch_indexer_throttle_acquire(timeout: int = 0, atomic: bool = True):
|
||||
"""
|
||||
Enable the task throttle flag for a delay.
|
||||
Uses redis locks if available to ensure atomic changes
|
||||
"""
|
||||
key = "document-batch-indexer-throttle"
|
||||
|
||||
# Redis is used as cache database (not in tests). Use the lock feature here
|
||||
# to ensure atomicity of changes to the throttle flag.
|
||||
if isinstance(cache, RedisCache) and atomic:
|
||||
with cache.locks(key):
|
||||
return batch_indexer_throttle_acquire(timeout, atomic=False)
|
||||
|
||||
# Use add() here :
|
||||
# - set the flag and returns true if not exist
|
||||
# - do nothing and return false if exist
|
||||
return cache.add(key, 1, timeout=timeout)
|
||||
|
||||
|
||||
@app.task
|
||||
def batch_document_indexer_task(timestamp):
|
||||
"""Celery Task : Sends indexation query for a batch of documents."""
|
||||
indexer = get_document_indexer()
|
||||
|
||||
if indexer:
|
||||
queryset = models.Document.objects.filter(
|
||||
Q(updated_at__gte=timestamp)
|
||||
| Q(deleted_at__gte=timestamp)
|
||||
| Q(ancestors_deleted_at__gte=timestamp)
|
||||
)
|
||||
|
||||
count = indexer.index(queryset)
|
||||
logger.info("Indexed %d documents", count)
|
||||
|
||||
|
||||
def trigger_batch_document_indexer(item):
|
||||
"""
|
||||
Trigger indexation task with debounce a delay set by the SEARCH_INDEXER_COUNTDOWN setting.
|
||||
|
||||
Args:
|
||||
document (Document): The document instance.
|
||||
"""
|
||||
countdown = int(settings.SEARCH_INDEXER_COUNTDOWN)
|
||||
|
||||
# DO NOT create a task if indexation if disabled
|
||||
if not settings.SEARCH_INDEXER_CLASS:
|
||||
return
|
||||
|
||||
if countdown > 0:
|
||||
# Each time this method is called during a countdown, we increment the
|
||||
# counter and each task decrease it, so the index be run only once.
|
||||
if batch_indexer_throttle_acquire(timeout=countdown):
|
||||
logger.info(
|
||||
"Add task for batch document indexation from updated_at=%s in %d seconds",
|
||||
item.updated_at.isoformat(),
|
||||
countdown,
|
||||
)
|
||||
|
||||
batch_document_indexer_task.apply_async(
|
||||
args=[item.updated_at], countdown=countdown
|
||||
)
|
||||
else:
|
||||
logger.info("Skip task for batch document %s indexation", item.pk)
|
||||
else:
|
||||
document_indexer_task.apply(args=[item.pk])
|
||||
@@ -0,0 +1,65 @@
|
||||
"""
|
||||
Unit test for `index` command.
|
||||
"""
|
||||
|
||||
from operator import itemgetter
|
||||
from unittest import mock
|
||||
|
||||
from django.core.management import CommandError, call_command
|
||||
from django.db import transaction
|
||||
|
||||
import pytest
|
||||
|
||||
from core import factories
|
||||
from core.services.search_indexers import SearchIndexer
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_index():
|
||||
"""Test the command `index` that run the Find app indexer for all the available documents."""
|
||||
user = factories.UserFactory()
|
||||
indexer = SearchIndexer()
|
||||
|
||||
with transaction.atomic():
|
||||
doc = factories.DocumentFactory()
|
||||
empty_doc = factories.DocumentFactory(title=None, content="")
|
||||
no_title_doc = factories.DocumentFactory(title=None)
|
||||
|
||||
factories.UserDocumentAccessFactory(document=doc, user=user)
|
||||
factories.UserDocumentAccessFactory(document=empty_doc, user=user)
|
||||
factories.UserDocumentAccessFactory(document=no_title_doc, user=user)
|
||||
|
||||
accesses = {
|
||||
str(doc.path): {"users": [user.sub]},
|
||||
str(empty_doc.path): {"users": [user.sub]},
|
||||
str(no_title_doc.path): {"users": [user.sub]},
|
||||
}
|
||||
|
||||
with mock.patch.object(SearchIndexer, "push") as mock_push:
|
||||
call_command("index")
|
||||
|
||||
push_call_args = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
# called once but with a batch of docs
|
||||
mock_push.assert_called_once()
|
||||
|
||||
assert sorted(push_call_args[0], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(doc, accesses),
|
||||
indexer.serialize_document(no_title_doc, accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_index_improperly_configured(indexer_settings):
|
||||
"""The command should raise an exception if the indexer is not configured"""
|
||||
indexer_settings.SEARCH_INDEXER_CLASS = None
|
||||
|
||||
with pytest.raises(CommandError) as err:
|
||||
call_command("index")
|
||||
|
||||
assert str(err.value) == "The indexer is not enabled or properly configured."
|
||||
@@ -24,3 +24,30 @@ def mock_user_teams():
|
||||
"core.models.User.teams", new_callable=mock.PropertyMock
|
||||
) as mock_teams:
|
||||
yield mock_teams
|
||||
|
||||
|
||||
@pytest.fixture(name="indexer_settings")
|
||||
def indexer_settings_fixture(settings):
|
||||
"""
|
||||
Setup valid settings for the document indexer. Clear the indexer cache.
|
||||
"""
|
||||
|
||||
# pylint: disable-next=import-outside-toplevel
|
||||
from core.services.search_indexers import ( # noqa: PLC0415
|
||||
get_document_indexer,
|
||||
)
|
||||
|
||||
get_document_indexer.cache_clear()
|
||||
|
||||
settings.SEARCH_INDEXER_CLASS = "core.services.search_indexers.SearchIndexer"
|
||||
settings.SEARCH_INDEXER_SECRET = "ThisIsAKeyForTest"
|
||||
settings.SEARCH_INDEXER_URL = "http://localhost:8081/api/v1.0/documents/index/"
|
||||
settings.SEARCH_INDEXER_QUERY_URL = (
|
||||
"http://localhost:8081/api/v1.0/documents/search/"
|
||||
)
|
||||
settings.SEARCH_INDEXER_COUNTDOWN = 1
|
||||
|
||||
yield settings
|
||||
|
||||
# clear cache to prevent issues with other tests
|
||||
get_document_indexer.cache_clear()
|
||||
|
||||
@@ -0,0 +1,425 @@
|
||||
"""
|
||||
Tests for Documents API endpoint in impress's core app: list
|
||||
"""
|
||||
|
||||
import random
|
||||
from json import loads as json_loads
|
||||
|
||||
from django.test import RequestFactory
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
from faker import Faker
|
||||
from rest_framework.test import APIClient
|
||||
|
||||
from core import factories, models
|
||||
from core.services.search_indexers import get_document_indexer
|
||||
|
||||
fake = Faker()
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def build_search_url(**kwargs):
|
||||
"""Build absolute uri for search endpoint with ORDERED query arguments"""
|
||||
return (
|
||||
RequestFactory()
|
||||
.get("/api/v1.0/documents/search/", dict(sorted(kwargs.items())))
|
||||
.build_absolute_uri()
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", models.LinkRoleChoices.values)
|
||||
@pytest.mark.parametrize("reach", models.LinkReachChoices.values)
|
||||
@responses.activate
|
||||
def test_api_documents_search_anonymous(reach, role, indexer_settings):
|
||||
"""
|
||||
Anonymous users should not be allowed to search documents whatever the
|
||||
link reach and link role
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = "http://find/api/v1.0/search"
|
||||
|
||||
factories.DocumentFactory(link_reach=reach, link_role=role)
|
||||
|
||||
# Find response
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"http://find/api/v1.0/search",
|
||||
json=[],
|
||||
status=200,
|
||||
)
|
||||
|
||||
response = APIClient().get("/api/v1.0/documents/search/", data={"q": "alpha"})
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {
|
||||
"count": 0,
|
||||
"next": None,
|
||||
"previous": None,
|
||||
"results": [],
|
||||
}
|
||||
|
||||
|
||||
def test_api_documents_search_endpoint_is_none(indexer_settings):
|
||||
"""
|
||||
Missing SEARCH_INDEXER_QUERY_URL, so the indexer is not properly configured.
|
||||
Should fallback on title filter
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = None
|
||||
|
||||
assert get_document_indexer() is None
|
||||
|
||||
user = factories.UserFactory()
|
||||
document = factories.DocumentFactory(title="alpha")
|
||||
access = factories.UserDocumentAccessFactory(document=document, user=user)
|
||||
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
response = client.get("/api/v1.0/documents/search/", data={"q": "alpha"})
|
||||
|
||||
assert response.status_code == 200
|
||||
content = response.json()
|
||||
results = content.pop("results")
|
||||
assert content == {
|
||||
"count": 1,
|
||||
"next": None,
|
||||
"previous": None,
|
||||
}
|
||||
assert len(results) == 1
|
||||
assert results[0] == {
|
||||
"id": str(document.id),
|
||||
"abilities": document.get_abilities(user),
|
||||
"ancestors_link_reach": None,
|
||||
"ancestors_link_role": None,
|
||||
"computed_link_reach": document.computed_link_reach,
|
||||
"computed_link_role": document.computed_link_role,
|
||||
"created_at": document.created_at.isoformat().replace("+00:00", "Z"),
|
||||
"creator": str(document.creator.id),
|
||||
"depth": 1,
|
||||
"excerpt": document.excerpt,
|
||||
"link_reach": document.link_reach,
|
||||
"link_role": document.link_role,
|
||||
"nb_accesses_ancestors": 1,
|
||||
"nb_accesses_direct": 1,
|
||||
"numchild": 0,
|
||||
"path": document.path,
|
||||
"title": document.title,
|
||||
"updated_at": document.updated_at.isoformat().replace("+00:00", "Z"),
|
||||
"deleted_at": None,
|
||||
"user_role": access.role,
|
||||
}
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_invalid_params(indexer_settings):
|
||||
"""Validate the format of documents as returned by the search view."""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = "http://find/api/v1.0/search"
|
||||
|
||||
user = factories.UserFactory()
|
||||
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
response = client.get("/api/v1.0/documents/search/")
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"q": ["This field is required."]}
|
||||
|
||||
response = client.get("/api/v1.0/documents/search/", data={"q": " "})
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"q": ["This field may not be blank."]}
|
||||
|
||||
response = client.get(
|
||||
"/api/v1.0/documents/search/", data={"q": "any", "page": "NaN"}
|
||||
)
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.json() == {"page": ["A valid integer is required."]}
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_api_documents_search_format(indexer_settings):
|
||||
"""Validate the format of documents as returned by the search view."""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = "http://find/api/v1.0/search"
|
||||
|
||||
assert get_document_indexer() is not None
|
||||
|
||||
user = factories.UserFactory()
|
||||
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
user_a, user_b, user_c = factories.UserFactory.create_batch(3)
|
||||
document = factories.DocumentFactory(
|
||||
title="alpha",
|
||||
users=(user_a, user_c),
|
||||
link_traces=(user, user_b),
|
||||
)
|
||||
access = factories.UserDocumentAccessFactory(document=document, user=user)
|
||||
|
||||
# Find response
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"http://find/api/v1.0/search",
|
||||
json=[
|
||||
{"_id": str(document.pk)},
|
||||
],
|
||||
status=200,
|
||||
)
|
||||
response = client.get("/api/v1.0/documents/search/", data={"q": "alpha"})
|
||||
|
||||
assert response.status_code == 200
|
||||
content = response.json()
|
||||
results = content.pop("results")
|
||||
assert content == {
|
||||
"count": 1,
|
||||
"next": None,
|
||||
"previous": None,
|
||||
}
|
||||
assert len(results) == 1
|
||||
assert results[0] == {
|
||||
"id": str(document.id),
|
||||
"abilities": document.get_abilities(user),
|
||||
"ancestors_link_reach": None,
|
||||
"ancestors_link_role": None,
|
||||
"computed_link_reach": document.computed_link_reach,
|
||||
"computed_link_role": document.computed_link_role,
|
||||
"created_at": document.created_at.isoformat().replace("+00:00", "Z"),
|
||||
"creator": str(document.creator.id),
|
||||
"depth": 1,
|
||||
"excerpt": document.excerpt,
|
||||
"link_reach": document.link_reach,
|
||||
"link_role": document.link_role,
|
||||
"nb_accesses_ancestors": 3,
|
||||
"nb_accesses_direct": 3,
|
||||
"numchild": 0,
|
||||
"path": document.path,
|
||||
"title": document.title,
|
||||
"updated_at": document.updated_at.isoformat().replace("+00:00", "Z"),
|
||||
"deleted_at": None,
|
||||
"user_role": access.role,
|
||||
}
|
||||
|
||||
|
||||
@responses.activate
|
||||
@pytest.mark.parametrize(
|
||||
"pagination, status, expected",
|
||||
(
|
||||
(
|
||||
{"page": 1, "page_size": 10},
|
||||
200,
|
||||
{
|
||||
"count": 10,
|
||||
"previous": None,
|
||||
"next": None,
|
||||
"range": (0, None),
|
||||
},
|
||||
),
|
||||
(
|
||||
{},
|
||||
200,
|
||||
{
|
||||
"count": 10,
|
||||
"previous": None,
|
||||
"next": None,
|
||||
"range": (0, None),
|
||||
"api_page_size": 21, # default page_size is 20
|
||||
},
|
||||
),
|
||||
(
|
||||
{"page": 2, "page_size": 10},
|
||||
404,
|
||||
{},
|
||||
),
|
||||
(
|
||||
{"page": 1, "page_size": 5},
|
||||
200,
|
||||
{
|
||||
"count": 10,
|
||||
"previous": None,
|
||||
"next": {"page": 2, "page_size": 5},
|
||||
"range": (0, 5),
|
||||
},
|
||||
),
|
||||
(
|
||||
{"page": 2, "page_size": 5},
|
||||
200,
|
||||
{
|
||||
"count": 10,
|
||||
"previous": {"page_size": 5},
|
||||
"next": None,
|
||||
"range": (5, None),
|
||||
},
|
||||
),
|
||||
({"page": 3, "page_size": 5}, 404, {}),
|
||||
),
|
||||
)
|
||||
def test_api_documents_search_pagination(
|
||||
indexer_settings, pagination, status, expected
|
||||
):
|
||||
"""Documents should be ordered by descending "score" by default"""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = "http://find/api/v1.0/search"
|
||||
|
||||
assert get_document_indexer() is not None
|
||||
|
||||
user = factories.UserFactory()
|
||||
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
docs = factories.DocumentFactory.create_batch(10, title="alpha", users=[user])
|
||||
|
||||
docs_by_uuid = {str(doc.pk): doc for doc in docs}
|
||||
api_results = [{"_id": id} for id in docs_by_uuid.keys()]
|
||||
|
||||
# reorder randomly to simulate score ordering
|
||||
random.shuffle(api_results)
|
||||
|
||||
# Find response
|
||||
# pylint: disable-next=assignment-from-none
|
||||
api_search = responses.add(
|
||||
responses.POST,
|
||||
"http://find/api/v1.0/search",
|
||||
json=api_results,
|
||||
status=200,
|
||||
)
|
||||
|
||||
response = client.get(
|
||||
"/api/v1.0/documents/search/",
|
||||
data={
|
||||
"q": "alpha",
|
||||
**pagination,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status
|
||||
|
||||
if response.status_code < 300:
|
||||
previous_url = (
|
||||
build_search_url(q="alpha", **expected["previous"])
|
||||
if expected["previous"]
|
||||
else None
|
||||
)
|
||||
next_url = (
|
||||
build_search_url(q="alpha", **expected["next"])
|
||||
if expected["next"]
|
||||
else None
|
||||
)
|
||||
start, end = expected["range"]
|
||||
|
||||
content = response.json()
|
||||
|
||||
assert content["count"] == expected["count"]
|
||||
assert content["previous"] == previous_url
|
||||
assert content["next"] == next_url
|
||||
|
||||
results = content.pop("results")
|
||||
|
||||
# The find api results ordering by score is kept
|
||||
assert [r["id"] for r in results] == [r["_id"] for r in api_results[start:end]]
|
||||
|
||||
# Check the query parameters.
|
||||
assert api_search.call_count == 1
|
||||
assert api_search.calls[0].response.status_code == 200
|
||||
assert json_loads(api_search.calls[0].request.body) == {
|
||||
"q": "alpha",
|
||||
"visited": [],
|
||||
"services": ["docs"],
|
||||
"nb_results": 50,
|
||||
"order_by": "updated_at",
|
||||
"order_direction": "desc",
|
||||
}
|
||||
|
||||
|
||||
@responses.activate
|
||||
@pytest.mark.parametrize(
|
||||
"pagination, status, expected",
|
||||
(
|
||||
(
|
||||
{"page": 1, "page_size": 10},
|
||||
200,
|
||||
{"count": 10, "previous": None, "next": None, "range": (0, None)},
|
||||
),
|
||||
(
|
||||
{},
|
||||
200,
|
||||
{"count": 10, "previous": None, "next": None, "range": (0, None)},
|
||||
),
|
||||
(
|
||||
{"page": 2, "page_size": 10},
|
||||
404,
|
||||
{},
|
||||
),
|
||||
(
|
||||
{"page": 1, "page_size": 5},
|
||||
200,
|
||||
{
|
||||
"count": 10,
|
||||
"previous": None,
|
||||
"next": {"page": 2, "page_size": 5},
|
||||
"range": (0, 5),
|
||||
},
|
||||
),
|
||||
(
|
||||
{"page": 2, "page_size": 5},
|
||||
200,
|
||||
{
|
||||
"count": 10,
|
||||
"previous": {"page_size": 5},
|
||||
"next": None,
|
||||
"range": (5, None),
|
||||
},
|
||||
),
|
||||
({"page": 3, "page_size": 5}, 404, {}),
|
||||
),
|
||||
)
|
||||
def test_api_documents_search_pagination_endpoint_is_none(
|
||||
indexer_settings, pagination, status, expected
|
||||
):
|
||||
"""Documents should be ordered by descending "-updated_at" by default"""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = None
|
||||
|
||||
assert get_document_indexer() is None
|
||||
|
||||
user = factories.UserFactory()
|
||||
|
||||
client = APIClient()
|
||||
client.force_login(user)
|
||||
|
||||
factories.DocumentFactory.create_batch(10, title="alpha", users=[user])
|
||||
|
||||
response = client.get(
|
||||
"/api/v1.0/documents/search/",
|
||||
data={
|
||||
"q": "alpha",
|
||||
**pagination,
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == status
|
||||
|
||||
if response.status_code < 300:
|
||||
previous_url = (
|
||||
build_search_url(q="alpha", **expected["previous"])
|
||||
if expected["previous"]
|
||||
else None
|
||||
)
|
||||
next_url = (
|
||||
build_search_url(q="alpha", **expected["next"])
|
||||
if expected["next"]
|
||||
else None
|
||||
)
|
||||
queryset = models.Document.objects.order_by("-updated_at")
|
||||
start, end = expected["range"]
|
||||
expected_results = [str(d.pk) for d in queryset[start:end]]
|
||||
|
||||
content = response.json()
|
||||
|
||||
assert content["count"] == expected["count"]
|
||||
assert content["previous"] == previous_url
|
||||
assert content["next"] == next_url
|
||||
|
||||
results = content.pop("results")
|
||||
|
||||
assert [r["id"] for r in results] == expected_results
|
||||
@@ -0,0 +1,441 @@
|
||||
"""
|
||||
Unit tests for the Document model
|
||||
"""
|
||||
# pylint: disable=too-many-lines
|
||||
|
||||
from operator import itemgetter
|
||||
from unittest import mock
|
||||
|
||||
from django.core.cache import cache
|
||||
from django.db import transaction
|
||||
|
||||
import pytest
|
||||
|
||||
from core import factories, models
|
||||
from core.services.search_indexers import SearchIndexer
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
def reset_batch_indexer_throttle():
|
||||
"""Reset throttle flag"""
|
||||
cache.delete("document-batch-indexer-throttle")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_throttle():
|
||||
"""Reset throttle flag before each test"""
|
||||
reset_batch_indexer_throttle()
|
||||
yield
|
||||
reset_batch_indexer_throttle()
|
||||
|
||||
|
||||
@mock.patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_models_documents_post_save_indexer(mock_push):
|
||||
"""Test indexation task on document creation"""
|
||||
with transaction.atomic():
|
||||
doc1, doc2, doc3 = factories.DocumentFactory.create_batch(3)
|
||||
|
||||
accesses = {}
|
||||
data = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
indexer = SearchIndexer()
|
||||
|
||||
assert len(data) == 1
|
||||
|
||||
# One call
|
||||
assert sorted(data[0], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(doc1, accesses),
|
||||
indexer.serialize_document(doc2, accesses),
|
||||
indexer.serialize_document(doc3, accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
# The throttle counters should be reset
|
||||
assert cache.get("document-batch-indexer-throttle") == 1
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_models_documents_post_save_indexer_no_batches(indexer_settings):
|
||||
"""Test indexation task on doculment creation, no throttle"""
|
||||
indexer_settings.SEARCH_INDEXER_COUNTDOWN = 0
|
||||
|
||||
with mock.patch.object(SearchIndexer, "push") as mock_push:
|
||||
with transaction.atomic():
|
||||
doc1, doc2, doc3 = factories.DocumentFactory.create_batch(3)
|
||||
|
||||
accesses = {}
|
||||
data = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
indexer = SearchIndexer()
|
||||
|
||||
# 3 calls
|
||||
assert len(data) == 3
|
||||
# one document per call
|
||||
assert [len(d) for d in data] == [1] * 3
|
||||
# all documents are indexed
|
||||
assert sorted([d[0] for d in data], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(doc1, accesses),
|
||||
indexer.serialize_document(doc2, accesses),
|
||||
indexer.serialize_document(doc3, accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
# The throttle counters should be reset
|
||||
assert cache.get("file-batch-indexer-throttle") is None
|
||||
|
||||
|
||||
@mock.patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_models_documents_post_save_indexer_not_configured(mock_push, indexer_settings):
|
||||
"""Task should not start an indexation when disabled"""
|
||||
indexer_settings.SEARCH_INDEXER_CLASS = None
|
||||
|
||||
user = factories.UserFactory()
|
||||
|
||||
with transaction.atomic():
|
||||
doc = factories.DocumentFactory()
|
||||
factories.UserDocumentAccessFactory(document=doc, user=user)
|
||||
|
||||
assert mock_push.assert_not_called
|
||||
|
||||
|
||||
@mock.patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_models_documents_post_save_indexer_wrongly_configured(
|
||||
mock_push, indexer_settings
|
||||
):
|
||||
"""Task should not start an indexation when disabled"""
|
||||
indexer_settings.SEARCH_INDEXER_URL = None
|
||||
|
||||
user = factories.UserFactory()
|
||||
|
||||
with transaction.atomic():
|
||||
doc = factories.DocumentFactory()
|
||||
factories.UserDocumentAccessFactory(document=doc, user=user)
|
||||
|
||||
assert mock_push.assert_not_called
|
||||
|
||||
|
||||
@mock.patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_models_documents_post_save_indexer_with_accesses(mock_push):
|
||||
"""Test indexation task on document creation"""
|
||||
user = factories.UserFactory()
|
||||
|
||||
with transaction.atomic():
|
||||
doc1, doc2, doc3 = factories.DocumentFactory.create_batch(3)
|
||||
|
||||
factories.UserDocumentAccessFactory(document=doc1, user=user)
|
||||
factories.UserDocumentAccessFactory(document=doc2, user=user)
|
||||
factories.UserDocumentAccessFactory(document=doc3, user=user)
|
||||
|
||||
accesses = {
|
||||
str(doc1.path): {"users": [user.sub]},
|
||||
str(doc2.path): {"users": [user.sub]},
|
||||
str(doc3.path): {"users": [user.sub]},
|
||||
}
|
||||
|
||||
data = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
indexer = SearchIndexer()
|
||||
|
||||
assert len(data) == 1
|
||||
assert sorted(data[0], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(doc1, accesses),
|
||||
indexer.serialize_document(doc2, accesses),
|
||||
indexer.serialize_document(doc3, accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
|
||||
@mock.patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_models_documents_post_save_indexer_deleted(mock_push):
|
||||
"""Indexation task on deleted or ancestor_deleted documents"""
|
||||
user = factories.UserFactory()
|
||||
|
||||
with transaction.atomic():
|
||||
doc = factories.DocumentFactory(
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED
|
||||
)
|
||||
main_doc = factories.DocumentFactory(
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED
|
||||
)
|
||||
child_doc = factories.DocumentFactory(
|
||||
parent=main_doc,
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED,
|
||||
)
|
||||
|
||||
factories.UserDocumentAccessFactory(document=doc, user=user)
|
||||
factories.UserDocumentAccessFactory(document=main_doc, user=user)
|
||||
factories.UserDocumentAccessFactory(document=child_doc, user=user)
|
||||
|
||||
# Manually reset the throttle flag here or the next indexation will be ignored for 1 second
|
||||
reset_batch_indexer_throttle()
|
||||
|
||||
with transaction.atomic():
|
||||
main_doc_deleted = models.Document.objects.get(pk=main_doc.pk)
|
||||
main_doc_deleted.soft_delete()
|
||||
|
||||
child_doc_deleted = models.Document.objects.get(pk=child_doc.pk)
|
||||
|
||||
main_doc_deleted.refresh_from_db()
|
||||
child_doc_deleted.refresh_from_db()
|
||||
|
||||
assert main_doc_deleted.deleted_at is not None
|
||||
assert child_doc_deleted.ancestors_deleted_at is not None
|
||||
|
||||
assert child_doc_deleted.deleted_at is None
|
||||
assert child_doc_deleted.ancestors_deleted_at is not None
|
||||
|
||||
accesses = {
|
||||
str(doc.path): {"users": [user.sub]},
|
||||
str(main_doc_deleted.path): {"users": [user.sub]},
|
||||
str(child_doc_deleted.path): {"users": [user.sub]},
|
||||
}
|
||||
|
||||
data = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
indexer = SearchIndexer()
|
||||
|
||||
assert len(data) == 2
|
||||
|
||||
# First indexation on document creation
|
||||
assert sorted(data[0], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(doc, accesses),
|
||||
indexer.serialize_document(main_doc, accesses),
|
||||
indexer.serialize_document(child_doc, accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
# Even deleted items are re-indexed : only update their status in the future
|
||||
assert sorted(data[1], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(main_doc_deleted, accesses), # soft_delete()
|
||||
indexer.serialize_document(child_doc_deleted, accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_models_documents_indexer_hard_deleted():
|
||||
"""Indexation task on hard deleted document"""
|
||||
user = factories.UserFactory()
|
||||
|
||||
with transaction.atomic():
|
||||
doc = factories.DocumentFactory(
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED
|
||||
)
|
||||
factories.UserDocumentAccessFactory(document=doc, user=user)
|
||||
|
||||
# Call task on deleted document.
|
||||
with mock.patch.object(SearchIndexer, "push") as mock_push:
|
||||
doc.delete()
|
||||
|
||||
# Hard delete document are not re-indexed.
|
||||
assert mock_push.assert_not_called
|
||||
|
||||
|
||||
@mock.patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_models_documents_post_save_indexer_restored(mock_push):
|
||||
"""Restart indexation task on restored documents"""
|
||||
user = factories.UserFactory()
|
||||
|
||||
with transaction.atomic():
|
||||
doc = factories.DocumentFactory(
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED
|
||||
)
|
||||
doc_deleted = factories.DocumentFactory(
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED
|
||||
)
|
||||
doc_ancestor_deleted = factories.DocumentFactory(
|
||||
parent=doc_deleted,
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED,
|
||||
)
|
||||
|
||||
factories.UserDocumentAccessFactory(document=doc, user=user)
|
||||
factories.UserDocumentAccessFactory(document=doc_deleted, user=user)
|
||||
factories.UserDocumentAccessFactory(document=doc_ancestor_deleted, user=user)
|
||||
|
||||
doc_deleted.soft_delete()
|
||||
|
||||
doc_deleted.refresh_from_db()
|
||||
doc_ancestor_deleted.refresh_from_db()
|
||||
|
||||
assert doc_deleted.deleted_at is not None
|
||||
assert doc_deleted.ancestors_deleted_at is not None
|
||||
|
||||
assert doc_ancestor_deleted.deleted_at is None
|
||||
assert doc_ancestor_deleted.ancestors_deleted_at is not None
|
||||
|
||||
# Manually reset the throttle flag here or the next indexation will be ignored for 1 second
|
||||
reset_batch_indexer_throttle()
|
||||
|
||||
with transaction.atomic():
|
||||
doc_restored = models.Document.objects.get(pk=doc_deleted.pk)
|
||||
doc_restored.restore()
|
||||
|
||||
doc_ancestor_restored = models.Document.objects.get(pk=doc_ancestor_deleted.pk)
|
||||
|
||||
assert doc_restored.deleted_at is None
|
||||
assert doc_restored.ancestors_deleted_at is None
|
||||
|
||||
assert doc_ancestor_restored.deleted_at is None
|
||||
assert doc_ancestor_restored.ancestors_deleted_at is None
|
||||
|
||||
accesses = {
|
||||
str(doc.path): {"users": [user.sub]},
|
||||
str(doc_deleted.path): {"users": [user.sub]},
|
||||
str(doc_ancestor_deleted.path): {"users": [user.sub]},
|
||||
}
|
||||
|
||||
data = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
indexer = SearchIndexer()
|
||||
|
||||
# All docs are re-indexed
|
||||
assert len(data) == 2
|
||||
|
||||
# First indexation on items creation & soft delete (in the same transaction)
|
||||
assert sorted(data[0], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(doc, accesses),
|
||||
indexer.serialize_document(doc_deleted, accesses),
|
||||
indexer.serialize_document(doc_ancestor_deleted, accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
# Restored items are re-indexed : only update their status in the future
|
||||
assert sorted(data[1], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(doc_restored, accesses), # restore()
|
||||
indexer.serialize_document(doc_ancestor_restored, accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_models_documents_post_save_indexer_throttle():
|
||||
"""Test indexation task skipping on document update"""
|
||||
indexer = SearchIndexer()
|
||||
user = factories.UserFactory()
|
||||
|
||||
with mock.patch.object(SearchIndexer, "push"):
|
||||
with transaction.atomic():
|
||||
docs = factories.DocumentFactory.create_batch(5, users=(user,))
|
||||
|
||||
accesses = {str(item.path): {"users": [user.sub]} for item in docs}
|
||||
|
||||
with mock.patch.object(SearchIndexer, "push") as mock_push:
|
||||
# Simulate 1 running task
|
||||
cache.set("document-batch-indexer-throttle", 1)
|
||||
|
||||
# save doc to trigger the indexer, but nothing should be done since
|
||||
# the flag is up
|
||||
with transaction.atomic():
|
||||
docs[0].save()
|
||||
docs[2].save()
|
||||
docs[3].save()
|
||||
|
||||
assert [call.args[0] for call in mock_push.call_args_list] == []
|
||||
|
||||
with mock.patch.object(SearchIndexer, "push") as mock_push:
|
||||
# No waiting task
|
||||
cache.delete("document-batch-indexer-throttle")
|
||||
|
||||
with transaction.atomic():
|
||||
docs[0].save()
|
||||
docs[2].save()
|
||||
docs[3].save()
|
||||
|
||||
data = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
# One call
|
||||
assert len(data) == 1
|
||||
|
||||
assert sorted(data[0], key=itemgetter("id")) == sorted(
|
||||
[
|
||||
indexer.serialize_document(docs[0], accesses),
|
||||
indexer.serialize_document(docs[2], accesses),
|
||||
indexer.serialize_document(docs[3], accesses),
|
||||
],
|
||||
key=itemgetter("id"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_models_documents_access_post_save_indexer():
|
||||
"""Test indexation task on DocumentAccess update"""
|
||||
users = factories.UserFactory.create_batch(3)
|
||||
|
||||
with mock.patch.object(SearchIndexer, "push"):
|
||||
with transaction.atomic():
|
||||
doc = factories.DocumentFactory(users=users)
|
||||
doc_accesses = models.DocumentAccess.objects.filter(document=doc).order_by(
|
||||
"user__sub"
|
||||
)
|
||||
|
||||
reset_batch_indexer_throttle()
|
||||
|
||||
with mock.patch.object(SearchIndexer, "push") as mock_push:
|
||||
with transaction.atomic():
|
||||
for doc_access in doc_accesses:
|
||||
doc_access.save()
|
||||
|
||||
data = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
# One call
|
||||
assert len(data) == 1
|
||||
|
||||
assert [d["id"] for d in data[0]] == [str(doc.pk)]
|
||||
|
||||
|
||||
@pytest.mark.django_db(transaction=True)
|
||||
def test_models_items_access_post_save_indexer_no_throttle(indexer_settings):
|
||||
"""Test indexation task on ItemAccess update, no throttle"""
|
||||
indexer_settings.SEARCH_INDEXER_COUNTDOWN = 0
|
||||
|
||||
users = factories.UserFactory.create_batch(3)
|
||||
|
||||
with transaction.atomic():
|
||||
doc = factories.DocumentFactory(users=users)
|
||||
doc_accesses = models.DocumentAccess.objects.filter(document=doc).order_by(
|
||||
"user__sub"
|
||||
)
|
||||
|
||||
reset_batch_indexer_throttle()
|
||||
|
||||
with mock.patch.object(SearchIndexer, "push") as mock_push:
|
||||
with transaction.atomic():
|
||||
for doc_access in doc_accesses:
|
||||
doc_access.save()
|
||||
|
||||
data = [call.args[0] for call in mock_push.call_args_list]
|
||||
|
||||
# 3 calls
|
||||
assert len(data) == 3
|
||||
# one document per call
|
||||
assert [len(d) for d in data] == [1] * 3
|
||||
# the same document is indexed 3 times
|
||||
assert [d[0]["id"] for d in data] == [str(doc.pk)] * 3
|
||||
@@ -0,0 +1,701 @@
|
||||
"""Tests for Documents search indexers"""
|
||||
|
||||
from functools import partial
|
||||
from json import dumps as json_dumps
|
||||
from unittest.mock import patch
|
||||
|
||||
from django.contrib.auth.models import AnonymousUser
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
from django.utils.module_loading import import_string
|
||||
|
||||
import pytest
|
||||
import responses
|
||||
from requests import HTTPError
|
||||
|
||||
from core import factories, models, utils
|
||||
from core.services.converter_services import (
|
||||
ServiceUnavailableError as YProviderServiceUnavailableError,
|
||||
)
|
||||
from core.services.search_indexers import (
|
||||
BaseDocumentIndexer,
|
||||
SearchIndexer,
|
||||
get_document_indexer,
|
||||
get_visited_document_ids_of,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.django_db
|
||||
|
||||
|
||||
class FakeDocumentIndexer(BaseDocumentIndexer):
|
||||
"""Fake indexer for test purpose"""
|
||||
|
||||
def serialize_document(self, document, accesses):
|
||||
return {}
|
||||
|
||||
def push(self, data):
|
||||
pass
|
||||
|
||||
def search_query(self, data, token):
|
||||
return {}
|
||||
|
||||
|
||||
def test_services_search_indexer_class_invalid(indexer_settings):
|
||||
"""
|
||||
Should raise RuntimeError if SEARCH_INDEXER_CLASS cannot be imported.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_CLASS = "unknown.Unknown"
|
||||
|
||||
assert get_document_indexer() is None
|
||||
|
||||
|
||||
def test_services_search_indexer_class(indexer_settings):
|
||||
"""
|
||||
Import indexer class defined in setting SEARCH_INDEXER_CLASS.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_CLASS = (
|
||||
"core.tests.test_services_search_indexers.FakeDocumentIndexer"
|
||||
)
|
||||
|
||||
assert isinstance(
|
||||
get_document_indexer(),
|
||||
import_string("core.tests.test_services_search_indexers.FakeDocumentIndexer"),
|
||||
)
|
||||
|
||||
|
||||
def test_services_search_indexer_is_configured(indexer_settings):
|
||||
"""
|
||||
Should return true only when the indexer class and other configuration settings
|
||||
are valid.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_CLASS = None
|
||||
|
||||
# None
|
||||
get_document_indexer.cache_clear()
|
||||
assert not get_document_indexer()
|
||||
|
||||
# Empty
|
||||
indexer_settings.SEARCH_INDEXER_CLASS = ""
|
||||
|
||||
get_document_indexer.cache_clear()
|
||||
assert not get_document_indexer()
|
||||
|
||||
# Valid class
|
||||
indexer_settings.SEARCH_INDEXER_CLASS = (
|
||||
"core.services.search_indexers.SearchIndexer"
|
||||
)
|
||||
|
||||
get_document_indexer.cache_clear()
|
||||
assert get_document_indexer() is not None
|
||||
|
||||
indexer_settings.SEARCH_INDEXER_URL = ""
|
||||
|
||||
# Invalid url
|
||||
get_document_indexer.cache_clear()
|
||||
assert not get_document_indexer()
|
||||
|
||||
|
||||
def test_services_search_indexer_url_is_none(indexer_settings):
|
||||
"""
|
||||
Indexer should raise RuntimeError if SEARCH_INDEXER_URL is None or empty.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_URL = None
|
||||
|
||||
with pytest.raises(ImproperlyConfigured) as exc_info:
|
||||
SearchIndexer()
|
||||
|
||||
assert "SEARCH_INDEXER_URL must be set in Django settings." in str(exc_info.value)
|
||||
|
||||
|
||||
def test_services_search_indexer_url_is_empty(indexer_settings):
|
||||
"""
|
||||
Indexer should raise RuntimeError if SEARCH_INDEXER_URL is empty string.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_URL = ""
|
||||
|
||||
with pytest.raises(ImproperlyConfigured) as exc_info:
|
||||
SearchIndexer()
|
||||
|
||||
assert "SEARCH_INDEXER_URL must be set in Django settings." in str(exc_info.value)
|
||||
|
||||
|
||||
def test_services_search_indexer_secret_is_none(indexer_settings):
|
||||
"""
|
||||
Indexer should raise RuntimeError if SEARCH_INDEXER_SECRET is None.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_SECRET = None
|
||||
|
||||
with pytest.raises(ImproperlyConfigured) as exc_info:
|
||||
SearchIndexer()
|
||||
|
||||
assert "SEARCH_INDEXER_SECRET must be set in Django settings." in str(
|
||||
exc_info.value
|
||||
)
|
||||
|
||||
|
||||
def test_services_search_indexer_secret_is_empty(indexer_settings):
|
||||
"""
|
||||
Indexer should raise RuntimeError if SEARCH_INDEXER_SECRET is empty string.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_SECRET = ""
|
||||
|
||||
with pytest.raises(ImproperlyConfigured) as exc_info:
|
||||
SearchIndexer()
|
||||
|
||||
assert "SEARCH_INDEXER_SECRET must be set in Django settings." in str(
|
||||
exc_info.value
|
||||
)
|
||||
|
||||
|
||||
def test_services_search_endpoint_is_none(indexer_settings):
|
||||
"""
|
||||
Indexer should raise RuntimeError if SEARCH_INDEXER_QUERY_URL is None.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = None
|
||||
|
||||
with pytest.raises(ImproperlyConfigured) as exc_info:
|
||||
SearchIndexer()
|
||||
|
||||
assert "SEARCH_INDEXER_QUERY_URL must be set in Django settings." in str(
|
||||
exc_info.value
|
||||
)
|
||||
|
||||
|
||||
def test_services_search_endpoint_is_empty(indexer_settings):
|
||||
"""
|
||||
Indexer should raise RuntimeError if SEARCH_INDEXER_QUERY_URL is empty.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = ""
|
||||
|
||||
with pytest.raises(ImproperlyConfigured) as exc_info:
|
||||
SearchIndexer()
|
||||
|
||||
assert "SEARCH_INDEXER_QUERY_URL must be set in Django settings." in str(
|
||||
exc_info.value
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
@patch("core.services.converter_services.YdocConverter.convert")
|
||||
def test_services_search_indexers_serialize_document_returns_expected_json(
|
||||
mock_convert,
|
||||
):
|
||||
"""
|
||||
It should serialize documents with correct metadata and access control.
|
||||
"""
|
||||
user_a, user_b = factories.UserFactory.create_batch(2)
|
||||
document = factories.DocumentFactory()
|
||||
factories.DocumentFactory(parent=document)
|
||||
|
||||
markdown_content = (
|
||||
f"## {document.title}\n{utils.base64_yjs_to_text(document.content)}"
|
||||
)
|
||||
|
||||
mock_convert.return_value = markdown_content
|
||||
|
||||
factories.UserDocumentAccessFactory(document=document, user=user_a)
|
||||
factories.UserDocumentAccessFactory(document=document, user=user_b)
|
||||
factories.TeamDocumentAccessFactory(document=document, team="team1")
|
||||
factories.TeamDocumentAccessFactory(document=document, team="team2")
|
||||
|
||||
accesses = {
|
||||
document.path: {
|
||||
"users": {str(user_a.sub), str(user_b.sub)},
|
||||
"teams": {"team1", "team2"},
|
||||
}
|
||||
}
|
||||
|
||||
indexer = SearchIndexer()
|
||||
result = indexer.serialize_document(document, accesses)
|
||||
|
||||
assert mock_convert.call_count == 1
|
||||
|
||||
assert set(result.pop("users")) == {str(user_a.sub), str(user_b.sub)}
|
||||
assert set(result.pop("groups")) == {"team1", "team2"}
|
||||
assert result == {
|
||||
"id": str(document.id),
|
||||
"title": document.title,
|
||||
"depth": 1,
|
||||
"path": document.path,
|
||||
"numchild": 1,
|
||||
"content": markdown_content,
|
||||
"mimetype": "text/markdown",
|
||||
"created_at": document.created_at.isoformat(),
|
||||
"updated_at": document.updated_at.isoformat(),
|
||||
"reach": document.link_reach,
|
||||
"size": len(markdown_content),
|
||||
"is_active": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
@patch("core.services.converter_services.YdocConverter.convert")
|
||||
def test_services_search_indexers_serialize_document_no_converter(
|
||||
mock_convert,
|
||||
):
|
||||
"""
|
||||
It should serialize documents with correct metadata and access control.
|
||||
"""
|
||||
user_a, user_b = factories.UserFactory.create_batch(2)
|
||||
document = factories.DocumentFactory()
|
||||
factories.DocumentFactory(parent=document)
|
||||
|
||||
mock_convert.side_effect = YProviderServiceUnavailableError()
|
||||
|
||||
text_content = utils.base64_yjs_to_text(document.content)
|
||||
|
||||
factories.UserDocumentAccessFactory(document=document, user=user_a)
|
||||
factories.UserDocumentAccessFactory(document=document, user=user_b)
|
||||
factories.TeamDocumentAccessFactory(document=document, team="team1")
|
||||
factories.TeamDocumentAccessFactory(document=document, team="team2")
|
||||
|
||||
accesses = {
|
||||
document.path: {
|
||||
"users": {str(user_a.sub), str(user_b.sub)},
|
||||
"teams": {"team1", "team2"},
|
||||
}
|
||||
}
|
||||
|
||||
indexer = SearchIndexer()
|
||||
result = indexer.serialize_document(document, accesses)
|
||||
|
||||
assert mock_convert.call_count == 1
|
||||
|
||||
assert set(result.pop("users")) == {str(user_a.sub), str(user_b.sub)}
|
||||
assert set(result.pop("groups")) == {"team1", "team2"}
|
||||
assert result == {
|
||||
"id": str(document.id),
|
||||
"title": document.title,
|
||||
"depth": 1,
|
||||
"path": document.path,
|
||||
"numchild": 1,
|
||||
"content": text_content,
|
||||
"mimetype": "text/markdown",
|
||||
"created_at": document.created_at.isoformat(),
|
||||
"updated_at": document.updated_at.isoformat(),
|
||||
"reach": document.link_reach,
|
||||
"size": len(text_content),
|
||||
"is_active": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_services_search_indexers_serialize_document_deleted():
|
||||
"""Deleted documents are marked as just in the serialized json."""
|
||||
parent = factories.DocumentFactory()
|
||||
document = factories.DocumentFactory(parent=parent)
|
||||
|
||||
parent.soft_delete()
|
||||
document.refresh_from_db()
|
||||
|
||||
indexer = SearchIndexer()
|
||||
result = indexer.serialize_document(document, {})
|
||||
|
||||
assert result["is_active"] is False
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_services_search_indexers_serialize_document_empty():
|
||||
"""Empty documents returns empty content in the serialized json."""
|
||||
document = factories.DocumentFactory(content="", title=None)
|
||||
|
||||
indexer = SearchIndexer()
|
||||
result = indexer.serialize_document(document, {})
|
||||
|
||||
assert result["content"] == ""
|
||||
assert result["title"] == ""
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_services_search_indexers_index_errors(indexer_settings):
|
||||
"""
|
||||
Documents indexing response handling on Find API HTTP errors.
|
||||
"""
|
||||
factories.DocumentFactory()
|
||||
|
||||
indexer_settings.SEARCH_INDEXER_URL = "http://app-find/api/v1.0/documents/index/"
|
||||
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"http://app-find/api/v1.0/documents/index/",
|
||||
status=401,
|
||||
body=json_dumps({"message": "Authentication failed."}),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPError):
|
||||
SearchIndexer().index()
|
||||
|
||||
|
||||
@patch.object(SearchIndexer, "push")
|
||||
def test_services_search_indexers_batches_pass_only_batch_accesses(
|
||||
mock_push, indexer_settings
|
||||
):
|
||||
"""
|
||||
Documents indexing should be processed in batches,
|
||||
and only the access data relevant to each batch should be used.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_BATCH_SIZE = 2
|
||||
documents = factories.DocumentFactory.create_batch(5)
|
||||
|
||||
# Attach a single user access to each document
|
||||
expected_user_subs = {}
|
||||
for document in documents:
|
||||
access = factories.UserDocumentAccessFactory(document=document)
|
||||
expected_user_subs[str(document.id)] = str(access.user.sub)
|
||||
|
||||
assert SearchIndexer().index() == 5
|
||||
|
||||
# Should be 3 batches: 2 + 2 + 1
|
||||
assert mock_push.call_count == 3
|
||||
|
||||
seen_doc_ids = set()
|
||||
|
||||
for call in mock_push.call_args_list:
|
||||
batch = call.args[0]
|
||||
assert isinstance(batch, list)
|
||||
|
||||
for doc_json in batch:
|
||||
doc_id = doc_json["id"]
|
||||
seen_doc_ids.add(doc_id)
|
||||
|
||||
# Only one user expected per document
|
||||
assert doc_json["users"] == [expected_user_subs[doc_id]]
|
||||
assert doc_json["groups"] == []
|
||||
|
||||
# Make sure all 5 documents were indexed
|
||||
assert seen_doc_ids == {str(d.id) for d in documents}
|
||||
|
||||
|
||||
@patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_services_search_indexers_batch_size_argument(mock_push):
|
||||
"""
|
||||
Documents indexing should be processed in batches,
|
||||
batch_size overrides SEARCH_INDEXER_BATCH_SIZE
|
||||
"""
|
||||
documents = factories.DocumentFactory.create_batch(5)
|
||||
|
||||
# Attach a single user access to each document
|
||||
expected_user_subs = {}
|
||||
for document in documents:
|
||||
access = factories.UserDocumentAccessFactory(document=document)
|
||||
expected_user_subs[str(document.id)] = str(access.user.sub)
|
||||
|
||||
assert SearchIndexer().index(batch_size=2) == 5
|
||||
|
||||
# Should be 3 batches: 2 + 2 + 1
|
||||
assert mock_push.call_count == 3
|
||||
|
||||
seen_doc_ids = set()
|
||||
|
||||
for call in mock_push.call_args_list:
|
||||
batch = call.args[0]
|
||||
assert isinstance(batch, list)
|
||||
|
||||
for doc_json in batch:
|
||||
doc_id = doc_json["id"]
|
||||
seen_doc_ids.add(doc_id)
|
||||
|
||||
# Only one user expected per document
|
||||
assert doc_json["users"] == [expected_user_subs[doc_id]]
|
||||
assert doc_json["groups"] == []
|
||||
|
||||
# Make sure all 5 documents were indexed
|
||||
assert seen_doc_ids == {str(d.id) for d in documents}
|
||||
|
||||
|
||||
@patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_services_search_indexers_ignore_empty_documents(mock_push):
|
||||
"""
|
||||
Documents indexing should be processed in batches,
|
||||
and only the access data relevant to each batch should be used.
|
||||
"""
|
||||
document = factories.DocumentFactory()
|
||||
factories.DocumentFactory(content="", title="")
|
||||
empty_title = factories.DocumentFactory(title="")
|
||||
empty_content = factories.DocumentFactory(content="")
|
||||
|
||||
assert SearchIndexer().index() == 3
|
||||
|
||||
assert mock_push.call_count == 1
|
||||
|
||||
# Make sure only not eempty documents are indexed
|
||||
results = {doc["id"] for doc in mock_push.call_args[0][0]}
|
||||
assert results == {
|
||||
str(d.id)
|
||||
for d in (
|
||||
document,
|
||||
empty_content,
|
||||
empty_title,
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@patch.object(SearchIndexer, "push")
|
||||
def test_services_search_indexers_skip_empty_batches(mock_push, indexer_settings):
|
||||
"""
|
||||
Documents indexing batch can be empty if all the docs are empty.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_BATCH_SIZE = 2
|
||||
|
||||
document = factories.DocumentFactory()
|
||||
|
||||
# Only empty docs
|
||||
factories.DocumentFactory.create_batch(5, content="", title="")
|
||||
|
||||
assert SearchIndexer().index() == 1
|
||||
assert mock_push.call_count == 1
|
||||
|
||||
results = [doc["id"] for doc in mock_push.call_args[0][0]]
|
||||
assert results == [str(document.id)]
|
||||
|
||||
|
||||
@patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_services_search_indexers_ancestors_link_reach(mock_push):
|
||||
"""Document accesses and reach should take into account ancestors link reaches."""
|
||||
great_grand_parent = factories.DocumentFactory(link_reach="restricted")
|
||||
grand_parent = factories.DocumentFactory(
|
||||
parent=great_grand_parent, link_reach="authenticated"
|
||||
)
|
||||
parent = factories.DocumentFactory(parent=grand_parent, link_reach="public")
|
||||
document = factories.DocumentFactory(parent=parent, link_reach="restricted")
|
||||
|
||||
assert SearchIndexer().index() == 4
|
||||
|
||||
results = {doc["id"]: doc for doc in mock_push.call_args[0][0]}
|
||||
assert len(results) == 4
|
||||
assert results[str(great_grand_parent.id)]["reach"] == "restricted"
|
||||
assert results[str(grand_parent.id)]["reach"] == "authenticated"
|
||||
assert results[str(parent.id)]["reach"] == "public"
|
||||
assert results[str(document.id)]["reach"] == "public"
|
||||
|
||||
|
||||
@patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_services_search_indexers_ancestors_users(mock_push):
|
||||
"""Document accesses and reach should include users from ancestors."""
|
||||
user_gp, user_p, user_d = factories.UserFactory.create_batch(3)
|
||||
|
||||
grand_parent = factories.DocumentFactory(users=[user_gp])
|
||||
parent = factories.DocumentFactory(parent=grand_parent, users=[user_p])
|
||||
document = factories.DocumentFactory(parent=parent, users=[user_d])
|
||||
|
||||
assert SearchIndexer().index() == 3
|
||||
|
||||
results = {doc["id"]: doc for doc in mock_push.call_args[0][0]}
|
||||
assert len(results) == 3
|
||||
assert results[str(grand_parent.id)]["users"] == [str(user_gp.sub)]
|
||||
assert set(results[str(parent.id)]["users"]) == {str(user_gp.sub), str(user_p.sub)}
|
||||
assert set(results[str(document.id)]["users"]) == {
|
||||
str(user_gp.sub),
|
||||
str(user_p.sub),
|
||||
str(user_d.sub),
|
||||
}
|
||||
|
||||
|
||||
@patch.object(SearchIndexer, "push")
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_services_search_indexers_ancestors_teams(mock_push):
|
||||
"""Document accesses and reach should include teams from ancestors."""
|
||||
grand_parent = factories.DocumentFactory(teams=["team_gp"])
|
||||
parent = factories.DocumentFactory(parent=grand_parent, teams=["team_p"])
|
||||
document = factories.DocumentFactory(parent=parent, teams=["team_d"])
|
||||
|
||||
assert SearchIndexer().index() == 3
|
||||
|
||||
results = {doc["id"]: doc for doc in mock_push.call_args[0][0]}
|
||||
assert len(results) == 3
|
||||
assert results[str(grand_parent.id)]["groups"] == ["team_gp"]
|
||||
assert set(results[str(parent.id)]["groups"]) == {"team_gp", "team_p"}
|
||||
assert set(results[str(document.id)]["groups"]) == {"team_gp", "team_p", "team_d"}
|
||||
|
||||
|
||||
@patch("requests.post")
|
||||
def test_push_uses_correct_url_and_data(mock_post, indexer_settings):
|
||||
"""
|
||||
push() should call requests.post with the correct URL from settings
|
||||
the timeout set to 10 seconds and the data as JSON.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_URL = "http://example.com/index"
|
||||
|
||||
indexer = SearchIndexer()
|
||||
sample_data = [{"id": "123", "title": "Test"}]
|
||||
|
||||
mock_response = mock_post.return_value
|
||||
mock_response.raise_for_status.return_value = None # No error
|
||||
|
||||
indexer.push(sample_data)
|
||||
|
||||
mock_post.assert_called_once()
|
||||
args, kwargs = mock_post.call_args
|
||||
|
||||
assert args[0] == indexer_settings.SEARCH_INDEXER_URL
|
||||
assert kwargs.get("json") == sample_data
|
||||
assert kwargs.get("timeout") == 10
|
||||
|
||||
|
||||
def test_get_visited_document_ids_of():
|
||||
"""
|
||||
get_visited_document_ids_of() returns the ids of the documents viewed
|
||||
by the user BUT without specific access configuration (like public ones)
|
||||
"""
|
||||
user = factories.UserFactory()
|
||||
other = factories.UserFactory()
|
||||
anonymous = AnonymousUser()
|
||||
queryset = models.Document.objects.all()
|
||||
|
||||
assert not get_visited_document_ids_of(queryset, anonymous)
|
||||
assert not get_visited_document_ids_of(queryset, user)
|
||||
|
||||
doc1, doc2, _ = factories.DocumentFactory.create_batch(3)
|
||||
|
||||
create_link = partial(models.LinkTrace.objects.create, user=user, is_masked=False)
|
||||
|
||||
create_link(document=doc1)
|
||||
create_link(document=doc2)
|
||||
|
||||
# The third document is not visited
|
||||
assert sorted(get_visited_document_ids_of(queryset, user)) == sorted(
|
||||
[str(doc1.pk), str(doc2.pk)]
|
||||
)
|
||||
|
||||
factories.UserDocumentAccessFactory(user=other, document=doc1)
|
||||
factories.UserDocumentAccessFactory(user=user, document=doc2)
|
||||
|
||||
# The second document have an access for the user
|
||||
assert get_visited_document_ids_of(queryset, user) == [str(doc1.pk)]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("indexer_settings")
|
||||
def test_get_visited_document_ids_of_deleted():
|
||||
"""
|
||||
get_visited_document_ids_of() returns the ids of the documents viewed
|
||||
by the user if they are not deleted.
|
||||
"""
|
||||
user = factories.UserFactory()
|
||||
anonymous = AnonymousUser()
|
||||
queryset = models.Document.objects.all()
|
||||
|
||||
assert not get_visited_document_ids_of(queryset, anonymous)
|
||||
assert not get_visited_document_ids_of(queryset, user)
|
||||
|
||||
doc = factories.DocumentFactory()
|
||||
doc_deleted = factories.DocumentFactory()
|
||||
doc_ancestor_deleted = factories.DocumentFactory(parent=doc_deleted)
|
||||
|
||||
create_link = partial(models.LinkTrace.objects.create, user=user, is_masked=False)
|
||||
|
||||
create_link(document=doc)
|
||||
create_link(document=doc_deleted)
|
||||
create_link(document=doc_ancestor_deleted)
|
||||
|
||||
# The all documents are visited
|
||||
assert sorted(get_visited_document_ids_of(queryset, user)) == sorted(
|
||||
[str(doc.pk), str(doc_deleted.pk), str(doc_ancestor_deleted.pk)]
|
||||
)
|
||||
|
||||
doc_deleted.soft_delete()
|
||||
|
||||
# Only the first document is not deleted
|
||||
assert get_visited_document_ids_of(queryset, user) == [str(doc.pk)]
|
||||
|
||||
|
||||
@responses.activate
|
||||
def test_services_search_indexers_search_errors(indexer_settings):
|
||||
"""
|
||||
Documents indexing response handling on Find API HTTP errors.
|
||||
"""
|
||||
factories.DocumentFactory()
|
||||
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_URL = (
|
||||
"http://app-find/api/v1.0/documents/search/"
|
||||
)
|
||||
|
||||
responses.add(
|
||||
responses.POST,
|
||||
"http://app-find/api/v1.0/documents/search/",
|
||||
status=401,
|
||||
body=json_dumps({"message": "Authentication failed."}),
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPError):
|
||||
SearchIndexer().search("alpha", token="mytoken")
|
||||
|
||||
|
||||
@patch("requests.post")
|
||||
def test_services_search_indexers_search(mock_post, indexer_settings):
|
||||
"""
|
||||
search() should call requests.post to SEARCH_INDEXER_QUERY_URL with the
|
||||
document ids from linktraces.
|
||||
"""
|
||||
user = factories.UserFactory()
|
||||
indexer = SearchIndexer()
|
||||
|
||||
mock_response = mock_post.return_value
|
||||
mock_response.raise_for_status.return_value = None # No error
|
||||
|
||||
doc1, doc2, _ = factories.DocumentFactory.create_batch(3)
|
||||
|
||||
create_link = partial(models.LinkTrace.objects.create, user=user, is_masked=False)
|
||||
|
||||
create_link(document=doc1)
|
||||
create_link(document=doc2)
|
||||
|
||||
visited = get_visited_document_ids_of(models.Document.objects.all(), user)
|
||||
|
||||
indexer.search("alpha", visited=visited, token="mytoken")
|
||||
|
||||
args, kwargs = mock_post.call_args
|
||||
|
||||
assert args[0] == indexer_settings.SEARCH_INDEXER_QUERY_URL
|
||||
|
||||
query_data = kwargs.get("json")
|
||||
assert query_data["q"] == "alpha"
|
||||
assert sorted(query_data["visited"]) == sorted([str(doc1.pk), str(doc2.pk)])
|
||||
assert query_data["services"] == ["docs"]
|
||||
assert query_data["nb_results"] == 50
|
||||
assert query_data["order_by"] == "updated_at"
|
||||
assert query_data["order_direction"] == "desc"
|
||||
|
||||
assert kwargs.get("headers") == {"Authorization": "Bearer mytoken"}
|
||||
assert kwargs.get("timeout") == 10
|
||||
|
||||
|
||||
@patch("requests.post")
|
||||
def test_services_search_indexers_search_nb_results(mock_post, indexer_settings):
|
||||
"""
|
||||
Find API call should have nb_results == SEARCH_INDEXER_QUERY_LIMIT
|
||||
or the given nb_results argument.
|
||||
"""
|
||||
indexer_settings.SEARCH_INDEXER_QUERY_LIMIT = 25
|
||||
|
||||
user = factories.UserFactory()
|
||||
indexer = SearchIndexer()
|
||||
|
||||
mock_response = mock_post.return_value
|
||||
mock_response.raise_for_status.return_value = None # No error
|
||||
|
||||
doc1, doc2, _ = factories.DocumentFactory.create_batch(3)
|
||||
|
||||
create_link = partial(models.LinkTrace.objects.create, user=user, is_masked=False)
|
||||
|
||||
create_link(document=doc1)
|
||||
create_link(document=doc2)
|
||||
|
||||
visited = get_visited_document_ids_of(models.Document.objects.all(), user)
|
||||
|
||||
indexer.search("alpha", visited=visited, token="mytoken")
|
||||
|
||||
args, kwargs = mock_post.call_args
|
||||
|
||||
assert args[0] == indexer_settings.SEARCH_INDEXER_QUERY_URL
|
||||
assert kwargs.get("json")["nb_results"] == 25
|
||||
|
||||
# The argument overrides the setting value
|
||||
indexer.search("alpha", visited=visited, token="mytoken", nb_results=109)
|
||||
|
||||
args, kwargs = mock_post.call_args
|
||||
|
||||
assert args[0] == indexer_settings.SEARCH_INDEXER_QUERY_URL
|
||||
assert kwargs.get("json")["nb_results"] == 109
|
||||
@@ -75,3 +75,28 @@ def test_utils_extract_attachments():
|
||||
base64_string = base64.b64encode(update).decode("utf-8")
|
||||
# image_key2 is missing the "/media/" part and shouldn't get extracted
|
||||
assert utils.extract_attachments(base64_string) == [image_key1, image_key3]
|
||||
|
||||
|
||||
def test_utils_get_ancestor_to_descendants_map_single_path():
|
||||
"""Test ancestor mapping of a single path."""
|
||||
paths = ["000100020005"]
|
||||
result = utils.get_ancestor_to_descendants_map(paths, steplen=4)
|
||||
|
||||
assert result == {
|
||||
"0001": {"000100020005"},
|
||||
"00010002": {"000100020005"},
|
||||
"000100020005": {"000100020005"},
|
||||
}
|
||||
|
||||
|
||||
def test_utils_get_ancestor_to_descendants_map_multiple_paths():
|
||||
"""Test ancestor mapping of multiple paths with shared prefixes."""
|
||||
paths = ["000100020005", "00010003"]
|
||||
result = utils.get_ancestor_to_descendants_map(paths, steplen=4)
|
||||
|
||||
assert result == {
|
||||
"0001": {"000100020005", "00010003"},
|
||||
"00010002": {"000100020005"},
|
||||
"000100020005": {"000100020005"},
|
||||
"00010003": {"00010003"},
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import base64
|
||||
import re
|
||||
from collections import defaultdict
|
||||
|
||||
import pycrdt
|
||||
from bs4 import BeautifulSoup
|
||||
@@ -9,6 +10,27 @@ from bs4 import BeautifulSoup
|
||||
from core import enums
|
||||
|
||||
|
||||
def get_ancestor_to_descendants_map(paths, steplen):
|
||||
"""
|
||||
Given a list of document paths, return a mapping of ancestor_path -> set of descendant_paths.
|
||||
|
||||
Each path is assumed to use materialized path format with fixed-length segments.
|
||||
|
||||
Args:
|
||||
paths (list of str): List of full document paths.
|
||||
steplen (int): Length of each path segment.
|
||||
|
||||
Returns:
|
||||
dict[str, set[str]]: Mapping from ancestor path to its descendant paths (including itself).
|
||||
"""
|
||||
ancestor_map = defaultdict(set)
|
||||
for path in paths:
|
||||
for i in range(steplen, len(path) + 1, steplen):
|
||||
ancestor = path[:i]
|
||||
ancestor_map[ancestor].add(path)
|
||||
return ancestor_map
|
||||
|
||||
|
||||
def filter_descendants(paths, root_paths, skip_sorting=False):
|
||||
"""
|
||||
Filters paths to keep only those that are descendants of any path in root_paths.
|
||||
|
||||
@@ -1,16 +1,19 @@
|
||||
# ruff: noqa: S311, S106
|
||||
"""create_demo management command"""
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from uuid import uuid4
|
||||
|
||||
from django import db
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand, CommandError
|
||||
|
||||
import pycrdt
|
||||
from faker import Faker
|
||||
|
||||
from core import models
|
||||
@@ -27,6 +30,16 @@ def random_true_with_probability(probability):
|
||||
return random.random() < probability
|
||||
|
||||
|
||||
def get_ydoc_for_text(text):
|
||||
"""Return a ydoc from plain text for demo purposes."""
|
||||
ydoc = pycrdt.Doc()
|
||||
paragraph = pycrdt.XmlElement("p", {}, [pycrdt.XmlText(text)])
|
||||
fragment = pycrdt.XmlFragment([paragraph])
|
||||
ydoc["document-store"] = fragment
|
||||
update = ydoc.get_update()
|
||||
return base64.b64encode(update).decode("utf-8")
|
||||
|
||||
|
||||
class BulkQueue:
|
||||
"""A utility class to create Django model instances in bulk by just pushing to a queue."""
|
||||
|
||||
@@ -48,7 +61,7 @@ class BulkQueue:
|
||||
self.queue[objects[0]._meta.model.__name__] = [] # noqa: SLF001
|
||||
|
||||
def push(self, obj):
|
||||
"""Add a model instance to queue to that it gets created in bulk."""
|
||||
"""Add a model instance to queue so that it gets created in bulk."""
|
||||
objects = self.queue[obj._meta.model.__name__] # noqa: SLF001
|
||||
objects.append(obj)
|
||||
if len(objects) > self.BATCH_SIZE:
|
||||
@@ -139,17 +152,19 @@ def create_demo(stdout):
|
||||
# pylint: disable=protected-access
|
||||
key = models.Document._int2str(i) # noqa: SLF001
|
||||
padding = models.Document.alphabet[0] * (models.Document.steplen - len(key))
|
||||
queue.push(
|
||||
models.Document(
|
||||
depth=1,
|
||||
path=f"{padding}{key}",
|
||||
creator_id=random.choice(users_ids),
|
||||
title=fake.sentence(nb_words=4),
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED
|
||||
if random_true_with_probability(0.5)
|
||||
else random.choice(models.LinkReachChoices.values),
|
||||
)
|
||||
title = fake.sentence(nb_words=4)
|
||||
document = models.Document(
|
||||
id=uuid4(),
|
||||
depth=1,
|
||||
path=f"{padding}{key}",
|
||||
creator_id=random.choice(users_ids),
|
||||
title=title,
|
||||
link_reach=models.LinkReachChoices.AUTHENTICATED
|
||||
if random_true_with_probability(0.5)
|
||||
else random.choice(models.LinkReachChoices.values),
|
||||
)
|
||||
document.save_content(get_ydoc_for_text(f"Content for {title:s}"))
|
||||
queue.push(document)
|
||||
|
||||
queue.flush()
|
||||
|
||||
|
||||
@@ -99,6 +99,31 @@ class Base(Configuration):
|
||||
}
|
||||
DEFAULT_AUTO_FIELD = "django.db.models.AutoField"
|
||||
|
||||
# Search
|
||||
SEARCH_INDEXER_CLASS = values.Value(
|
||||
default=None,
|
||||
environ_name="SEARCH_INDEXER_CLASS",
|
||||
environ_prefix=None,
|
||||
)
|
||||
SEARCH_INDEXER_BATCH_SIZE = values.IntegerValue(
|
||||
default=100_000, environ_name="SEARCH_INDEXER_BATCH_SIZE", environ_prefix=None
|
||||
)
|
||||
SEARCH_INDEXER_URL = values.Value(
|
||||
default=None, environ_name="SEARCH_INDEXER_URL", environ_prefix=None
|
||||
)
|
||||
SEARCH_INDEXER_COUNTDOWN = values.IntegerValue(
|
||||
default=1, environ_name="SEARCH_INDEXER_COUNTDOWN", environ_prefix=None
|
||||
)
|
||||
SEARCH_INDEXER_SECRET = values.Value(
|
||||
default=None, environ_name="SEARCH_INDEXER_SECRET", environ_prefix=None
|
||||
)
|
||||
SEARCH_INDEXER_QUERY_URL = values.Value(
|
||||
default=None, environ_name="SEARCH_INDEXER_QUERY_URL", environ_prefix=None
|
||||
)
|
||||
SEARCH_INDEXER_QUERY_LIMIT = values.PositiveIntegerValue(
|
||||
default=50, environ_name="SEARCH_INDEXER_QUERY_LIMIT", environ_prefix=None
|
||||
)
|
||||
|
||||
# Static files (CSS, JavaScript, Images)
|
||||
STATIC_URL = "/static/"
|
||||
STATIC_ROOT = os.path.join(DATA_DIR, "static")
|
||||
@@ -680,12 +705,6 @@ class Base(Configuration):
|
||||
environ_prefix=None,
|
||||
)
|
||||
|
||||
# DocSpec API microservice
|
||||
DOCSPEC_API_URL = values.Value(
|
||||
environ_name="DOCSPEC_API_URL",
|
||||
environ_prefix=None
|
||||
)
|
||||
|
||||
# Conversion endpoint
|
||||
CONVERSION_API_ENDPOINT = values.Value(
|
||||
default="convert",
|
||||
|
||||
@@ -806,6 +806,12 @@ test.describe('Doc Editor', () => {
|
||||
});
|
||||
await expect(interlinkChild1).toBeVisible({ timeout: 10000 });
|
||||
await expect(interlinkChild1.locator('svg').first()).toBeVisible();
|
||||
|
||||
await page.keyboard.press('@');
|
||||
|
||||
await page.keyboard.press('Escape');
|
||||
|
||||
await expect(editor.getByText('@')).toBeVisible();
|
||||
});
|
||||
|
||||
test('it checks multiple big doc scroll to the top', async ({
|
||||
|
||||
@@ -31,7 +31,7 @@ test.describe('Doc Export', () => {
|
||||
|
||||
await expect(page.getByTestId('modal-export-title')).toBeVisible();
|
||||
await expect(
|
||||
page.getByText('Download your document in a .docx or .pdf format.'),
|
||||
page.getByText('Download your document in a .docx, .odt or .pdf format.'),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByRole('combobox', { name: 'Template' }),
|
||||
@@ -142,6 +142,51 @@ test.describe('Doc Export', () => {
|
||||
expect(download.suggestedFilename()).toBe(`${randomDoc}.docx`);
|
||||
});
|
||||
|
||||
test('it exports the doc to odt', async ({ page, browserName }) => {
|
||||
const [randomDoc] = await createDoc(page, 'doc-editor-odt', browserName, 1);
|
||||
|
||||
await verifyDocName(page, randomDoc);
|
||||
|
||||
await page.locator('.ProseMirror.bn-editor').click();
|
||||
await page.locator('.ProseMirror.bn-editor').fill('Hello World ODT');
|
||||
|
||||
await page.keyboard.press('Enter');
|
||||
await page.locator('.bn-block-outer').last().fill('/');
|
||||
await page.getByText('Resizable image with caption').click();
|
||||
|
||||
const fileChooserPromise = page.waitForEvent('filechooser');
|
||||
await page.getByText('Upload image').click();
|
||||
|
||||
const fileChooser = await fileChooserPromise;
|
||||
await fileChooser.setFiles(path.join(__dirname, 'assets/test.svg'));
|
||||
|
||||
const image = page
|
||||
.locator('.--docs--editor-container img.bn-visual-media')
|
||||
.first();
|
||||
|
||||
await expect(image).toBeVisible();
|
||||
|
||||
await page
|
||||
.getByRole('button', {
|
||||
name: 'Export the document',
|
||||
})
|
||||
.click();
|
||||
|
||||
await page.getByRole('combobox', { name: 'Format' }).click();
|
||||
await page.getByRole('option', { name: 'Odt' }).click();
|
||||
|
||||
await expect(page.getByTestId('doc-export-download-button')).toBeVisible();
|
||||
|
||||
const downloadPromise = page.waitForEvent('download', (download) => {
|
||||
return download.suggestedFilename().includes(`${randomDoc}.odt`);
|
||||
});
|
||||
|
||||
void page.getByTestId('doc-export-download-button').click();
|
||||
|
||||
const download = await downloadPromise;
|
||||
expect(download.suggestedFilename()).toBe(`${randomDoc}.odt`);
|
||||
});
|
||||
|
||||
/**
|
||||
* This test tell us that the export to pdf is working with images
|
||||
* but it does not tell us if the images are being displayed correctly
|
||||
@@ -442,4 +487,68 @@ test.describe('Doc Export', () => {
|
||||
const pdfText = await pdfParse.getText();
|
||||
expect(pdfText.text).toContain(randomDoc);
|
||||
});
|
||||
|
||||
test('it exports the doc with interlinking to odt', async ({
|
||||
page,
|
||||
browserName,
|
||||
}) => {
|
||||
const [randomDoc] = await createDoc(
|
||||
page,
|
||||
'export-interlinking-odt',
|
||||
browserName,
|
||||
1,
|
||||
);
|
||||
|
||||
await verifyDocName(page, randomDoc);
|
||||
|
||||
const { name: docChild } = await createRootSubPage(
|
||||
page,
|
||||
browserName,
|
||||
'export-interlink-child-odt',
|
||||
);
|
||||
|
||||
await verifyDocName(page, docChild);
|
||||
|
||||
await page.locator('.bn-block-outer').last().fill('/');
|
||||
await page.getByText('Link a doc').first().click();
|
||||
|
||||
const input = page.locator(
|
||||
"span[data-inline-content-type='interlinkingSearchInline'] input",
|
||||
);
|
||||
const searchContainer = page.locator('.quick-search-container');
|
||||
|
||||
await input.fill('export-interlink');
|
||||
|
||||
await expect(searchContainer).toBeVisible();
|
||||
await expect(searchContainer.getByText(randomDoc)).toBeVisible();
|
||||
|
||||
// We are in docChild, we want to create a link to randomDoc (parent)
|
||||
await searchContainer.getByText(randomDoc).click();
|
||||
|
||||
// Search the interlinking link in the editor (not in the document tree)
|
||||
const editor = page.locator('.ProseMirror.bn-editor');
|
||||
const interlink = editor.getByRole('button', {
|
||||
name: randomDoc,
|
||||
});
|
||||
|
||||
await expect(interlink).toBeVisible();
|
||||
|
||||
await page
|
||||
.getByRole('button', {
|
||||
name: 'Export the document',
|
||||
})
|
||||
.click();
|
||||
|
||||
await page.getByRole('combobox', { name: 'Format' }).click();
|
||||
await page.getByRole('option', { name: 'Odt' }).click();
|
||||
|
||||
const downloadPromise = page.waitForEvent('download', (download) => {
|
||||
return download.suggestedFilename().includes(`${docChild}.odt`);
|
||||
});
|
||||
|
||||
void page.getByTestId('doc-export-download-button').click();
|
||||
|
||||
const download = await downloadPromise;
|
||||
expect(download.suggestedFilename()).toBe(`${docChild}.odt`);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
"@blocknote/react": "0.41.1",
|
||||
"@blocknote/xl-docx-exporter": "0.41.1",
|
||||
"@blocknote/xl-multi-column": "0.41.1",
|
||||
"@blocknote/xl-odt-exporter": "0.41.1",
|
||||
"@blocknote/xl-pdf-exporter": "0.41.1",
|
||||
"@dnd-kit/core": "6.3.1",
|
||||
"@dnd-kit/modifiers": "9.0.0",
|
||||
|
||||
@@ -12,6 +12,7 @@ import { css } from 'styled-components';
|
||||
|
||||
import { Box, BoxButton, BoxProps, DropButton, Icon, Text } from '@/components';
|
||||
import { useCunninghamTheme } from '@/cunningham';
|
||||
import { useKeyboardAction } from '@/hooks';
|
||||
|
||||
import { useDropdownKeyboardNav } from './hook/useDropdownKeyboardNav';
|
||||
|
||||
@@ -57,6 +58,7 @@ export const DropdownMenu = ({
|
||||
testId,
|
||||
}: PropsWithChildren<DropdownMenuProps>) => {
|
||||
const { spacingsTokens, colorsTokens } = useCunninghamTheme();
|
||||
const keyboardAction = useKeyboardAction();
|
||||
const [isOpen, setIsOpen] = useState(opened ?? false);
|
||||
const [focusedIndex, setFocusedIndex] = useState(-1);
|
||||
const blockButtonRef = useRef<HTMLDivElement>(null);
|
||||
@@ -93,6 +95,14 @@ export const DropdownMenu = ({
|
||||
}
|
||||
}, [isOpen, options]);
|
||||
|
||||
const triggerOption = useCallback(
|
||||
(option: DropdownMenuOption) => {
|
||||
onOpenChange?.(false);
|
||||
void option.callback?.();
|
||||
},
|
||||
[onOpenChange],
|
||||
);
|
||||
|
||||
if (disabled) {
|
||||
return children;
|
||||
}
|
||||
@@ -170,9 +180,9 @@ export const DropdownMenu = ({
|
||||
onClick={(event) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
onOpenChange?.(false);
|
||||
void option.callback?.();
|
||||
triggerOption(option);
|
||||
}}
|
||||
onKeyDown={keyboardAction(() => triggerOption(option))}
|
||||
key={option.label}
|
||||
$align="center"
|
||||
$justify="space-between"
|
||||
|
||||
@@ -186,6 +186,7 @@ export const BlockNoteEditor = ({ doc, provider }: BlockNoteEditorProps) => {
|
||||
formattingToolbar={false}
|
||||
slashMenu={false}
|
||||
theme="light"
|
||||
aria-label={t('Document editor')}
|
||||
>
|
||||
<BlockNoteSuggestionMenu />
|
||||
<BlockNoteToolbar />
|
||||
@@ -200,6 +201,7 @@ interface BlockNoteReaderProps {
|
||||
|
||||
export const BlockNoteReader = ({ initialContent }: BlockNoteReaderProps) => {
|
||||
const { setEditor } = useEditorStore();
|
||||
const { t } = useTranslation();
|
||||
const editor = useCreateBlockNote(
|
||||
{
|
||||
collaboration: {
|
||||
@@ -231,6 +233,7 @@ export const BlockNoteReader = ({ initialContent }: BlockNoteReaderProps) => {
|
||||
editor={editor}
|
||||
editable={false}
|
||||
theme="light"
|
||||
aria-label={t('Document version viewer')}
|
||||
formattingToolbar={false}
|
||||
slashMenu={false}
|
||||
/>
|
||||
|
||||
+2
-2
@@ -22,8 +22,8 @@ import { Box, Icon } from '@/components';
|
||||
import { DocsBlockNoteEditor } from '../../types';
|
||||
|
||||
const PDFBlockStyle = createGlobalStyle`
|
||||
.bn-block-content[data-content-type="pdf"] {
|
||||
width: fit-content;
|
||||
.bn-block-content[data-content-type="pdf"] .bn-file-block-content-wrapper[style*="fit-content"] {
|
||||
width: 100% !important;
|
||||
}
|
||||
`;
|
||||
|
||||
|
||||
+53
-44
@@ -3,6 +3,7 @@ import {
|
||||
StyleSchema,
|
||||
} from '@blocknote/core';
|
||||
import { useBlockNoteEditor } from '@blocknote/react';
|
||||
import type { KeyboardEvent } from 'react';
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { css } from 'styled-components';
|
||||
@@ -99,6 +100,55 @@ export const SearchPage = ({
|
||||
}, 100);
|
||||
}, [inputRef]);
|
||||
|
||||
const closeSearch = (insertContent: string) => {
|
||||
updateInlineContent({
|
||||
type: 'interlinkingSearchInline',
|
||||
props: {
|
||||
disabled: true,
|
||||
trigger,
|
||||
},
|
||||
});
|
||||
|
||||
contentRef(null);
|
||||
editor.focus();
|
||||
editor.insertInlineContent([insertContent]);
|
||||
};
|
||||
|
||||
const handleKeyDown = (e: KeyboardEvent<HTMLInputElement>) => {
|
||||
if (e.key === 'Escape') {
|
||||
e.preventDefault();
|
||||
// Keep the trigger character ('@' or '/') in the editor when closing with Escape
|
||||
closeSearch(trigger);
|
||||
} else if (e.key === 'Backspace' && search.length === 0) {
|
||||
e.preventDefault();
|
||||
closeSearch('');
|
||||
} else if (e.key === 'ArrowDown' || e.key === 'ArrowUp') {
|
||||
// Allow arrow keys to be handled by the command menu for navigation
|
||||
const commandList = e.currentTarget
|
||||
.closest('.inline-content')
|
||||
?.nextElementSibling?.querySelector('[cmdk-list]');
|
||||
|
||||
// Create a synthetic keyboard event for the command menu
|
||||
const syntheticEvent = new KeyboardEvent('keydown', {
|
||||
key: e.key,
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
});
|
||||
commandList?.dispatchEvent(syntheticEvent);
|
||||
e.preventDefault();
|
||||
} else if (e.key === 'Enter') {
|
||||
// Handle Enter key to select the currently highlighted item
|
||||
const selectedItem = e.currentTarget
|
||||
.closest('.inline-content')
|
||||
?.nextElementSibling?.querySelector(
|
||||
'[cmdk-item][data-selected="true"]',
|
||||
) as HTMLElement;
|
||||
|
||||
selectedItem?.click();
|
||||
e.preventDefault();
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<Box as="span" $position="relative">
|
||||
<Box
|
||||
@@ -124,50 +174,7 @@ export const SearchPage = ({
|
||||
const value = (e.target as HTMLInputElement).value;
|
||||
setSearch(value);
|
||||
}}
|
||||
onKeyDown={(e) => {
|
||||
if (
|
||||
(e.key === 'Backspace' && search.length === 0) ||
|
||||
e.key === 'Escape'
|
||||
) {
|
||||
e.preventDefault();
|
||||
|
||||
updateInlineContent({
|
||||
type: 'interlinkingSearchInline',
|
||||
props: {
|
||||
disabled: true,
|
||||
trigger,
|
||||
},
|
||||
});
|
||||
|
||||
contentRef(null);
|
||||
editor.focus();
|
||||
editor.insertInlineContent(['']);
|
||||
} else if (e.key === 'ArrowDown' || e.key === 'ArrowUp') {
|
||||
// Allow arrow keys to be handled by the command menu for navigation
|
||||
const commandList = e.currentTarget
|
||||
.closest('.inline-content')
|
||||
?.nextElementSibling?.querySelector('[cmdk-list]');
|
||||
|
||||
// Create a synthetic keyboard event for the command menu
|
||||
const syntheticEvent = new KeyboardEvent('keydown', {
|
||||
key: e.key,
|
||||
bubbles: true,
|
||||
cancelable: true,
|
||||
});
|
||||
commandList?.dispatchEvent(syntheticEvent);
|
||||
e.preventDefault();
|
||||
} else if (e.key === 'Enter') {
|
||||
// Handle Enter key to select the currently highlighted item
|
||||
const selectedItem = e.currentTarget
|
||||
.closest('.inline-content')
|
||||
?.nextElementSibling?.querySelector(
|
||||
'[cmdk-item][data-selected="true"]',
|
||||
) as HTMLElement;
|
||||
|
||||
selectedItem?.click();
|
||||
e.preventDefault();
|
||||
}
|
||||
}}
|
||||
onKeyDown={handleKeyDown}
|
||||
/>
|
||||
</Box>
|
||||
<Box
|
||||
@@ -224,6 +231,8 @@ export const SearchPage = ({
|
||||
},
|
||||
});
|
||||
|
||||
contentRef(null);
|
||||
|
||||
editor.insertInlineContent([
|
||||
{
|
||||
type: 'interlinkingLinkInline',
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import React from 'react';
|
||||
|
||||
import { DocsExporterODT } from '../types';
|
||||
import { odtRegisterParagraphStyleForBlock } from '../utils';
|
||||
|
||||
export const blockMappingCalloutODT: DocsExporterODT['mappings']['blockMapping']['callout'] =
|
||||
(block, exporter) => {
|
||||
// Map callout to paragraph with emoji prefix
|
||||
const emoji = block.props.emoji || '💡';
|
||||
|
||||
// Transform inline content (text, bold, links, etc.)
|
||||
const inlineContent = exporter.transformInlineContent(block.content);
|
||||
|
||||
// Resolve background and alignment → create a dedicated paragraph style
|
||||
const styleName = odtRegisterParagraphStyleForBlock(
|
||||
exporter,
|
||||
{
|
||||
backgroundColor: block.props.backgroundColor,
|
||||
textAlignment: block.props.textAlignment,
|
||||
},
|
||||
{ paddingCm: 0.42 },
|
||||
);
|
||||
|
||||
return React.createElement(
|
||||
'text:p',
|
||||
{
|
||||
'text:style-name': styleName,
|
||||
},
|
||||
`${emoji} `,
|
||||
...inlineContent,
|
||||
);
|
||||
};
|
||||
@@ -50,9 +50,9 @@ export const blockMappingImageDocx: DocsExporterDocx['mappings']['blockMapping']
|
||||
|
||||
const { width, height } = dimensions;
|
||||
|
||||
if (previewWidth && previewWidth > MAX_WIDTH) {
|
||||
previewWidth = MAX_WIDTH;
|
||||
}
|
||||
// Ensure the final width never exceeds MAX_WIDTH to prevent images
|
||||
// from overflowing the page width in the exported document
|
||||
const finalWidth = Math.min(previewWidth || width, MAX_WIDTH);
|
||||
|
||||
return [
|
||||
new Paragraph({
|
||||
@@ -71,8 +71,8 @@ export const blockMappingImageDocx: DocsExporterDocx['mappings']['blockMapping']
|
||||
}
|
||||
: undefined,
|
||||
transformation: {
|
||||
width: previewWidth || width,
|
||||
height: ((previewWidth || width) / width) * height,
|
||||
width: finalWidth,
|
||||
height: (finalWidth / width) * height,
|
||||
},
|
||||
}),
|
||||
],
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import React from 'react';
|
||||
|
||||
import { DocsExporterODT } from '../types';
|
||||
import { convertSvgToPng, odtRegisterParagraphStyleForBlock } from '../utils';
|
||||
|
||||
const MAX_WIDTH = 600;
|
||||
|
||||
export const blockMappingImageODT: DocsExporterODT['mappings']['blockMapping']['image'] =
|
||||
async (block, exporter) => {
|
||||
try {
|
||||
const blob = await exporter.resolveFile(block.props.url);
|
||||
|
||||
if (!blob || !blob.type) {
|
||||
console.warn(`Failed to resolve image: ${block.props.url}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
let pngConverted: string | undefined;
|
||||
let dimensions: { width: number; height: number } | undefined;
|
||||
let previewWidth = block.props.previewWidth || undefined;
|
||||
|
||||
if (!blob.type.includes('image')) {
|
||||
console.warn(`Not an image type: ${blob.type}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (blob.type.includes('svg')) {
|
||||
const svgText = await blob.text();
|
||||
const FALLBACK_SIZE = 536;
|
||||
previewWidth = previewWidth || blob.size || FALLBACK_SIZE;
|
||||
pngConverted = await convertSvgToPng(svgText, previewWidth);
|
||||
const img = new Image();
|
||||
img.src = pngConverted;
|
||||
await new Promise((resolve) => {
|
||||
img.onload = () => {
|
||||
dimensions = { width: img.width, height: img.height };
|
||||
resolve(null);
|
||||
};
|
||||
});
|
||||
} else {
|
||||
dimensions = await getImageDimensions(blob);
|
||||
}
|
||||
|
||||
if (!dimensions) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const { width, height } = dimensions;
|
||||
|
||||
if (previewWidth && previewWidth > MAX_WIDTH) {
|
||||
previewWidth = MAX_WIDTH;
|
||||
}
|
||||
|
||||
// Convert image to base64 for ODT embedding
|
||||
const arrayBuffer = pngConverted
|
||||
? await (await fetch(pngConverted)).arrayBuffer()
|
||||
: await blob.arrayBuffer();
|
||||
const base64 = btoa(
|
||||
Array.from(new Uint8Array(arrayBuffer))
|
||||
.map((byte) => String.fromCharCode(byte))
|
||||
.join(''),
|
||||
);
|
||||
|
||||
const finalWidth = previewWidth || width;
|
||||
const finalHeight = ((previewWidth || width) / width) * height;
|
||||
|
||||
const baseParagraphProps = {
|
||||
backgroundColor: block.props.backgroundColor,
|
||||
textAlignment: block.props.textAlignment,
|
||||
};
|
||||
|
||||
const paragraphStyleName = odtRegisterParagraphStyleForBlock(
|
||||
exporter,
|
||||
baseParagraphProps,
|
||||
{ paddingCm: 0 },
|
||||
);
|
||||
|
||||
// Convert pixels to cm (ODT uses cm for dimensions)
|
||||
const widthCm = finalWidth / 37.795275591;
|
||||
const heightCm = finalHeight / 37.795275591;
|
||||
|
||||
// Create ODT image structure using React.createElement
|
||||
const frame = React.createElement(
|
||||
'text:p',
|
||||
{
|
||||
'text:style-name': paragraphStyleName,
|
||||
},
|
||||
React.createElement(
|
||||
'draw:frame',
|
||||
{
|
||||
'draw:name': `Image${Date.now()}`,
|
||||
'text:anchor-type': 'as-char',
|
||||
'svg:width': `${widthCm}cm`,
|
||||
'svg:height': `${heightCm}cm`,
|
||||
},
|
||||
React.createElement(
|
||||
'draw:image',
|
||||
{
|
||||
xlinkType: 'simple',
|
||||
xlinkShow: 'embed',
|
||||
xlinkActuate: 'onLoad',
|
||||
},
|
||||
React.createElement('office:binary-data', {}, base64),
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
// Add caption if present
|
||||
if (block.props.caption) {
|
||||
const captionStyleName = odtRegisterParagraphStyleForBlock(
|
||||
exporter,
|
||||
baseParagraphProps,
|
||||
{ paddingCm: 0, parentStyleName: 'Caption' },
|
||||
);
|
||||
|
||||
return [
|
||||
frame,
|
||||
React.createElement(
|
||||
'text:p',
|
||||
{ 'text:style-name': captionStyleName },
|
||||
block.props.caption,
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
return frame;
|
||||
} catch (error) {
|
||||
console.error(`Error processing image for ODT export:`, error);
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
async function getImageDimensions(blob: Blob) {
|
||||
if (typeof window !== 'undefined') {
|
||||
const bmp = await createImageBitmap(blob);
|
||||
const { width, height } = bmp;
|
||||
bmp.close();
|
||||
return { width, height };
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,14 @@
|
||||
export * from './calloutDocx';
|
||||
export * from './calloutODT';
|
||||
export * from './calloutPDF';
|
||||
export * from './headingPDF';
|
||||
export * from './imageDocx';
|
||||
export * from './imageODT';
|
||||
export * from './imagePDF';
|
||||
export * from './paragraphPDF';
|
||||
export * from './quoteDocx';
|
||||
export * from './quotePDF';
|
||||
export * from './tablePDF';
|
||||
export * from './uploadLoaderPDF';
|
||||
export * from './uploadLoaderDocx';
|
||||
export * from './uploadLoaderODT';
|
||||
export * from './uploadLoaderPDF';
|
||||
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
import React from 'react';
|
||||
|
||||
import { DocsExporterODT } from '../types';
|
||||
|
||||
export const blockMappingUploadLoaderODT: DocsExporterODT['mappings']['blockMapping']['uploadLoader'] =
|
||||
(block) => {
|
||||
// Map uploadLoader to paragraph with information text
|
||||
const information = block.props.information || '';
|
||||
const type = block.props.type || 'loading';
|
||||
const prefix = type === 'warning' ? '⚠️ ' : '⏳ ';
|
||||
|
||||
return React.createElement(
|
||||
'text:p',
|
||||
{ 'text:style-name': 'Text_20_body' },
|
||||
`${prefix}${information}`,
|
||||
);
|
||||
};
|
||||
@@ -1,4 +1,5 @@
|
||||
import { DOCXExporter } from '@blocknote/xl-docx-exporter';
|
||||
import { ODTExporter } from '@blocknote/xl-odt-exporter';
|
||||
import { PDFExporter } from '@blocknote/xl-pdf-exporter';
|
||||
import {
|
||||
Button,
|
||||
@@ -23,12 +24,14 @@ import { Doc, useTrans } from '@/docs/doc-management';
|
||||
import { exportCorsResolveFileUrl } from '../api/exportResolveFileUrl';
|
||||
import { TemplatesOrdering, useTemplates } from '../api/useTemplates';
|
||||
import { docxDocsSchemaMappings } from '../mappingDocx';
|
||||
import { odtDocsSchemaMappings } from '../mappingODT';
|
||||
import { pdfDocsSchemaMappings } from '../mappingPDF';
|
||||
import { downloadFile } from '../utils';
|
||||
|
||||
enum DocDownloadFormat {
|
||||
PDF = 'pdf',
|
||||
DOCX = 'docx',
|
||||
ODT = 'odt',
|
||||
}
|
||||
|
||||
interface ModalExportProps {
|
||||
@@ -124,7 +127,7 @@ export const ModalExport = ({ onClose, doc }: ModalExportProps) => {
|
||||
: rawPdfDocument;
|
||||
|
||||
blobExport = await pdf(pdfDocument).toBlob();
|
||||
} else {
|
||||
} else if (format === DocDownloadFormat.DOCX) {
|
||||
const exporter = new DOCXExporter(editor.schema, docxDocsSchemaMappings, {
|
||||
resolveFileUrl: async (url) => exportCorsResolveFileUrl(doc.id, url),
|
||||
});
|
||||
@@ -133,6 +136,16 @@ export const ModalExport = ({ onClose, doc }: ModalExportProps) => {
|
||||
documentOptions: { title: documentTitle },
|
||||
sectionOptions: {},
|
||||
});
|
||||
} else if (format === DocDownloadFormat.ODT) {
|
||||
const exporter = new ODTExporter(editor.schema, odtDocsSchemaMappings, {
|
||||
resolveFileUrl: async (url) => exportCorsResolveFileUrl(doc.id, url),
|
||||
});
|
||||
|
||||
blobExport = await exporter.toODTDocument(exportDocument);
|
||||
} else {
|
||||
toast(t('The export failed'), VariantType.ERROR);
|
||||
setIsExporting(false);
|
||||
return;
|
||||
}
|
||||
|
||||
downloadFile(blobExport, `${filename}.${format}`);
|
||||
@@ -213,7 +226,7 @@ export const ModalExport = ({ onClose, doc }: ModalExportProps) => {
|
||||
className="--docs--modal-export-content"
|
||||
>
|
||||
<Text $variation="600" $size="sm" as="p">
|
||||
{t('Download your document in a .docx or .pdf format.')}
|
||||
{t('Download your document in a .docx, .odt or .pdf format.')}
|
||||
</Text>
|
||||
<Select
|
||||
clearable={false}
|
||||
@@ -231,6 +244,7 @@ export const ModalExport = ({ onClose, doc }: ModalExportProps) => {
|
||||
label={t('Format')}
|
||||
options={[
|
||||
{ label: t('Docx'), value: DocDownloadFormat.DOCX },
|
||||
{ label: t('ODT'), value: DocDownloadFormat.ODT },
|
||||
{ label: t('PDF'), value: DocDownloadFormat.PDF },
|
||||
]}
|
||||
value={format}
|
||||
|
||||
@@ -1,2 +1,3 @@
|
||||
export * from './interlinkingLinkPDF';
|
||||
export * from './interlinkingLinkDocx';
|
||||
export * from './interlinkingLinkODT';
|
||||
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
import React from 'react';
|
||||
|
||||
import { DocsExporterODT } from '../types';
|
||||
|
||||
export const inlineContentMappingInterlinkingLinkODT: DocsExporterODT['mappings']['inlineContentMapping']['interlinkingLinkInline'] =
|
||||
(inline) => {
|
||||
const url = window.location.origin + inline.props.url;
|
||||
const title = inline.props.title;
|
||||
|
||||
// Create ODT hyperlink using React.createElement to avoid TypeScript JSX namespace issues
|
||||
// Uses the same structure as BlockNote's default link mapping
|
||||
return React.createElement(
|
||||
'text:a',
|
||||
{
|
||||
xlinkType: 'simple',
|
||||
'text:style-name': 'Internet_20_link',
|
||||
'office:target-frame-name': '_top',
|
||||
xlinkShow: 'replace',
|
||||
xlinkHref: url,
|
||||
},
|
||||
`📄${title}`,
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
import { odtDefaultSchemaMappings } from '@blocknote/xl-odt-exporter';
|
||||
|
||||
import {
|
||||
blockMappingCalloutODT,
|
||||
blockMappingImageODT,
|
||||
blockMappingUploadLoaderODT,
|
||||
} from './blocks-mapping';
|
||||
import { inlineContentMappingInterlinkingLinkODT } from './inline-content-mapping';
|
||||
import { DocsExporterODT } from './types';
|
||||
|
||||
// Align default inline mappings to our editor inline schema without using `any`
|
||||
const baseInlineMappings =
|
||||
odtDefaultSchemaMappings.inlineContentMapping as unknown as DocsExporterODT['mappings']['inlineContentMapping'];
|
||||
|
||||
export const odtDocsSchemaMappings: DocsExporterODT['mappings'] = {
|
||||
...odtDefaultSchemaMappings,
|
||||
blockMapping: {
|
||||
...odtDefaultSchemaMappings.blockMapping,
|
||||
callout: blockMappingCalloutODT,
|
||||
image: blockMappingImageODT,
|
||||
// We're reusing the file block mapping for PDF blocks
|
||||
// The types don't match exactly but the implementation is compatible
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
pdf: odtDefaultSchemaMappings.blockMapping.file as any,
|
||||
uploadLoader: blockMappingUploadLoaderODT,
|
||||
},
|
||||
|
||||
inlineContentMapping: {
|
||||
...baseInlineMappings,
|
||||
interlinkingSearchInline: () => null,
|
||||
interlinkingLinkInline: inlineContentMappingInterlinkingLinkODT,
|
||||
},
|
||||
styleMapping: {
|
||||
...odtDefaultSchemaMappings.styleMapping,
|
||||
},
|
||||
};
|
||||
@@ -51,3 +51,13 @@ export type DocsExporterDocx = Exporter<
|
||||
IRunPropertiesOptions,
|
||||
TextRun
|
||||
>;
|
||||
|
||||
export type DocsExporterODT = Exporter<
|
||||
DocsBlockSchema,
|
||||
DocsInlineContentSchema,
|
||||
DocsStyleSchema,
|
||||
React.ReactNode,
|
||||
React.ReactNode,
|
||||
Record<string, string>,
|
||||
React.ReactNode
|
||||
>;
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
} from '@blocknote/core';
|
||||
import { Canvg } from 'canvg';
|
||||
import { IParagraphOptions, ShadingType } from 'docx';
|
||||
import React from 'react';
|
||||
|
||||
export function downloadFile(blob: Blob, filename: string) {
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
@@ -98,3 +99,76 @@ export function docxBlockPropsToStyles(
|
||||
})(),
|
||||
};
|
||||
}
|
||||
|
||||
// ODT helpers
|
||||
type OdtExporterLike = {
|
||||
options?: { colors?: typeof COLORS_DEFAULT };
|
||||
registerStyle: (fn: (name: string) => React.ReactNode) => string;
|
||||
};
|
||||
|
||||
function isOdtExporterLike(value: unknown): value is OdtExporterLike {
|
||||
return (
|
||||
!!value &&
|
||||
typeof (value as { registerStyle?: unknown }).registerStyle === 'function'
|
||||
);
|
||||
}
|
||||
|
||||
export function odtRegisterParagraphStyleForBlock(
|
||||
exporter: unknown,
|
||||
props: Partial<DefaultProps>,
|
||||
options?: { paddingCm?: number; parentStyleName?: string },
|
||||
) {
|
||||
if (!isOdtExporterLike(exporter)) {
|
||||
throw new Error('Invalid ODT exporter: missing registerStyle');
|
||||
}
|
||||
|
||||
const colors = exporter.options?.colors;
|
||||
|
||||
const bgColorHex =
|
||||
props.backgroundColor && props.backgroundColor !== 'default' && colors
|
||||
? colors[props.backgroundColor].background
|
||||
: undefined;
|
||||
|
||||
const textColorHex =
|
||||
props.textColor && props.textColor !== 'default' && colors
|
||||
? colors[props.textColor].text
|
||||
: undefined;
|
||||
|
||||
const foTextAlign =
|
||||
!props.textAlignment || props.textAlignment === 'left'
|
||||
? 'start'
|
||||
: props.textAlignment === 'center'
|
||||
? 'center'
|
||||
: props.textAlignment === 'right'
|
||||
? 'end'
|
||||
: 'justify';
|
||||
|
||||
const paddingCm = options?.paddingCm ?? 0.42; // ~1rem (16px)
|
||||
const parentStyleName = options?.parentStyleName;
|
||||
|
||||
// registerStyle is available on ODT exporter; call through with React elements
|
||||
const styleName = exporter.registerStyle((name: string) =>
|
||||
React.createElement(
|
||||
'style:style',
|
||||
{
|
||||
'style:name': name,
|
||||
'style:family': 'paragraph',
|
||||
...(parentStyleName
|
||||
? { 'style:parent-style-name': parentStyleName }
|
||||
: {}),
|
||||
},
|
||||
React.createElement('style:paragraph-properties', {
|
||||
'fo:text-align': foTextAlign,
|
||||
'fo:padding': `${paddingCm}cm`,
|
||||
...(bgColorHex ? { 'fo:background-color': bgColorHex } : {}),
|
||||
}),
|
||||
textColorHex
|
||||
? React.createElement('style:text-properties', {
|
||||
'fo:color': textColorHex,
|
||||
})
|
||||
: undefined,
|
||||
),
|
||||
);
|
||||
|
||||
return styleName;
|
||||
}
|
||||
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import React from 'react';
|
||||
import { describe, expect, test, vi } from 'vitest';
|
||||
|
||||
import { AppWrapper } from '@/tests/utils';
|
||||
|
||||
// Force mobile layout so the children count is rendered
|
||||
vi.mock('@/stores', () => ({
|
||||
useResponsiveStore: () => ({ isDesktop: false }),
|
||||
}));
|
||||
|
||||
// Provide stable mocks for hooks used by the component
|
||||
vi.mock('../../doc-management', async () => {
|
||||
const actual = await vi.importActual<any>('../../doc-management');
|
||||
return {
|
||||
...actual,
|
||||
useTrans: () => ({ transRole: vi.fn((r) => String(r)) }),
|
||||
useIsCollaborativeEditable: () => ({ isEditable: true }),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock('@/core', () => ({
|
||||
useConfig: () => ({ data: {} }),
|
||||
}));
|
||||
|
||||
vi.mock('@/hook', () => ({
|
||||
useDate: () => ({
|
||||
relativeDate: () => 'yesterday',
|
||||
calculateDaysLeft: () => 5,
|
||||
}),
|
||||
}));
|
||||
|
||||
import { DocHeaderInfo } from '../components/DocHeaderInfo';
|
||||
|
||||
describe('DocHeaderInfo', () => {
|
||||
test('renders the number of sub-documents when numchild is provided (mobile layout)', () => {
|
||||
const doc = {
|
||||
numchild: 3,
|
||||
updated_at: new Date().toISOString(),
|
||||
} as any;
|
||||
|
||||
render(<DocHeaderInfo doc={doc} />, { wrapper: AppWrapper });
|
||||
|
||||
expect(screen.getByText(/Contains 3 sub-documents/i)).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
+1
-1
@@ -44,7 +44,7 @@ describe('DocToolBox - Licence', () => {
|
||||
await userEvent.click(optionsButton);
|
||||
expect(
|
||||
await screen.findByText(
|
||||
'Download your document in a .docx or .pdf format.',
|
||||
'Download your document in a .docx, .odt or .pdf format.',
|
||||
),
|
||||
).toBeInTheDocument();
|
||||
}, 10000);
|
||||
|
||||
@@ -1,23 +1,20 @@
|
||||
import { useTranslation } from 'react-i18next';
|
||||
|
||||
import { Box, HorizontalSeparator, Text } from '@/components';
|
||||
import { useConfig } from '@/core';
|
||||
import { Box, HorizontalSeparator } from '@/components';
|
||||
import { useCunninghamTheme } from '@/cunningham';
|
||||
import {
|
||||
Doc,
|
||||
LinkReach,
|
||||
Role,
|
||||
getDocLinkReach,
|
||||
useIsCollaborativeEditable,
|
||||
useTrans,
|
||||
} from '@/docs/doc-management';
|
||||
import { useDate } from '@/hook';
|
||||
import { useResponsiveStore } from '@/stores';
|
||||
|
||||
import { AlertNetwork } from './AlertNetwork';
|
||||
import { AlertPublic } from './AlertPublic';
|
||||
import { AlertRestore } from './AlertRestore';
|
||||
import { BoutonShare } from './BoutonShare';
|
||||
import { DocHeaderInfo } from './DocHeaderInfo';
|
||||
import { DocTitle } from './DocTitle';
|
||||
import { DocToolBox } from './DocToolBox';
|
||||
|
||||
@@ -29,27 +26,11 @@ export const DocHeader = ({ doc }: DocHeaderProps) => {
|
||||
const { spacingsTokens } = useCunninghamTheme();
|
||||
const { isDesktop } = useResponsiveStore();
|
||||
const { t } = useTranslation();
|
||||
const { transRole } = useTrans();
|
||||
const { isEditable } = useIsCollaborativeEditable(doc);
|
||||
const docIsPublic = getDocLinkReach(doc) === LinkReach.PUBLIC;
|
||||
const docIsAuth = getDocLinkReach(doc) === LinkReach.AUTHENTICATED;
|
||||
const { relativeDate, calculateDaysLeft } = useDate();
|
||||
const { data: config } = useConfig();
|
||||
const isDeletedDoc = !!doc.deleted_at;
|
||||
|
||||
let dateToDisplay = t('Last update: {{update}}', {
|
||||
update: relativeDate(doc.updated_at),
|
||||
});
|
||||
|
||||
if (config?.TRASHBIN_CUTOFF_DAYS && doc.deleted_at) {
|
||||
const daysLeft = calculateDaysLeft(
|
||||
doc.deleted_at,
|
||||
config.TRASHBIN_CUTOFF_DAYS,
|
||||
);
|
||||
|
||||
dateToDisplay = `${t('Days remaining:')} ${daysLeft} ${t('days', { count: daysLeft })}`;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Box
|
||||
@@ -80,33 +61,8 @@ export const DocHeader = ({ doc }: DocHeaderProps) => {
|
||||
>
|
||||
<Box $gap={spacingsTokens['3xs']} $overflow="auto">
|
||||
<DocTitle doc={doc} />
|
||||
|
||||
<Box $direction="row">
|
||||
{isDesktop && (
|
||||
<>
|
||||
<Text
|
||||
$variation="600"
|
||||
$size="s"
|
||||
$weight="bold"
|
||||
$theme={isEditable ? 'greyscale' : 'warning'}
|
||||
>
|
||||
{transRole(
|
||||
isEditable
|
||||
? doc.user_role || doc.link_role
|
||||
: Role.READER,
|
||||
)}
|
||||
·
|
||||
</Text>
|
||||
<Text $variation="600" $size="s">
|
||||
{dateToDisplay}
|
||||
</Text>
|
||||
</>
|
||||
)}
|
||||
{!isDesktop && (
|
||||
<Text $variation="400" $size="s">
|
||||
{dateToDisplay}
|
||||
</Text>
|
||||
)}
|
||||
<DocHeaderInfo doc={doc} />
|
||||
</Box>
|
||||
</Box>
|
||||
{!isDeletedDoc && <DocToolBox doc={doc} />}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { t } from 'i18next';
|
||||
import React from 'react';
|
||||
|
||||
import { Text } from '@/components';
|
||||
import { useConfig } from '@/core';
|
||||
import { useDate } from '@/hook';
|
||||
import { useResponsiveStore } from '@/stores';
|
||||
|
||||
import {
|
||||
Doc,
|
||||
Role,
|
||||
useIsCollaborativeEditable,
|
||||
useTrans,
|
||||
} from '../../doc-management';
|
||||
|
||||
interface DocHeaderInfoProps {
|
||||
doc: Doc;
|
||||
}
|
||||
|
||||
export const DocHeaderInfo = ({ doc }: DocHeaderInfoProps) => {
|
||||
const { isDesktop } = useResponsiveStore();
|
||||
const { transRole } = useTrans();
|
||||
const { isEditable } = useIsCollaborativeEditable(doc);
|
||||
const { relativeDate, calculateDaysLeft } = useDate();
|
||||
const { data: config } = useConfig();
|
||||
|
||||
const childrenCount = doc.numchild ?? 0;
|
||||
|
||||
const relativeOnly = relativeDate(doc.updated_at);
|
||||
|
||||
let dateToDisplay = t('Last update: {{update}}', {
|
||||
update: relativeOnly,
|
||||
});
|
||||
|
||||
if (config?.TRASHBIN_CUTOFF_DAYS && doc.deleted_at) {
|
||||
const daysLeft = calculateDaysLeft(
|
||||
doc.deleted_at,
|
||||
config.TRASHBIN_CUTOFF_DAYS,
|
||||
);
|
||||
|
||||
dateToDisplay = `${t('Days remaining:')} ${daysLeft} ${t('days', { count: daysLeft })}`;
|
||||
}
|
||||
|
||||
const hasChildren = childrenCount > 0;
|
||||
|
||||
if (isDesktop) {
|
||||
return (
|
||||
<>
|
||||
<Text
|
||||
$variation="600"
|
||||
$size="s"
|
||||
$weight="bold"
|
||||
$theme={isEditable ? 'greyscale' : 'warning'}
|
||||
>
|
||||
{transRole(isEditable ? doc.user_role || doc.link_role : Role.READER)}
|
||||
·
|
||||
</Text>
|
||||
<Text $variation="600" $size="s">
|
||||
{dateToDisplay}
|
||||
</Text>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<Text $variation="400" $size="s">
|
||||
{hasChildren ? relativeOnly : dateToDisplay}
|
||||
</Text>
|
||||
{hasChildren && (
|
||||
<Text $variation="400" $size="s">
|
||||
•
|
||||
{t('Contains {{count}} sub-documents', {
|
||||
count: childrenCount,
|
||||
})}
|
||||
</Text>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
+37
-8
@@ -1,16 +1,19 @@
|
||||
import {
|
||||
Button,
|
||||
ButtonElement,
|
||||
Modal,
|
||||
ModalSize,
|
||||
VariantType,
|
||||
useToastProvider,
|
||||
} from '@openfun/cunningham-react';
|
||||
import { useRouter } from 'next/router';
|
||||
import { useEffect, useRef } from 'react';
|
||||
import { Trans, useTranslation } from 'react-i18next';
|
||||
|
||||
import { Box, ButtonCloseModal, Text, TextErrors } from '@/components';
|
||||
import { useConfig } from '@/core';
|
||||
import { KEY_LIST_DOC_TRASHBIN } from '@/docs/docs-grid';
|
||||
import { useKeyboardAction } from '@/hooks';
|
||||
|
||||
import { KEY_LIST_DOC } from '../api/useDocs';
|
||||
import { useRemoveDoc } from '../api/useRemoveDoc';
|
||||
@@ -34,6 +37,7 @@ export const ModalRemoveDoc = ({
|
||||
const trashBinCutoffDays = config?.TRASHBIN_CUTOFF_DAYS || 30;
|
||||
const { push } = useRouter();
|
||||
const { hasChildren } = useDocUtils(doc);
|
||||
const cancelButtonRef = useRef<ButtonElement>(null);
|
||||
const {
|
||||
mutate: removeDoc,
|
||||
isError,
|
||||
@@ -57,20 +61,47 @@ export const ModalRemoveDoc = ({
|
||||
},
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
const TIMEOUT_MODAL_MOUNTING = 100;
|
||||
const timeoutId = setTimeout(() => {
|
||||
const buttonElement = cancelButtonRef.current;
|
||||
if (buttonElement) {
|
||||
buttonElement.focus();
|
||||
}
|
||||
}, TIMEOUT_MODAL_MOUNTING);
|
||||
|
||||
return () => clearTimeout(timeoutId);
|
||||
}, []);
|
||||
|
||||
const keyboardAction = useKeyboardAction();
|
||||
|
||||
const handleClose = () => {
|
||||
onClose();
|
||||
};
|
||||
|
||||
const handleDelete = () => {
|
||||
removeDoc({ docId: doc.id });
|
||||
};
|
||||
|
||||
const handleCloseKeyDown = keyboardAction(handleClose);
|
||||
const handleDeleteKeyDown = keyboardAction(handleDelete);
|
||||
|
||||
return (
|
||||
<Modal
|
||||
isOpen
|
||||
closeOnClickOutside
|
||||
hideCloseButton
|
||||
onClose={() => onClose()}
|
||||
onClose={handleClose}
|
||||
aria-describedby="modal-remove-doc-title"
|
||||
rightActions={
|
||||
<>
|
||||
<Button
|
||||
ref={cancelButtonRef}
|
||||
aria-label={t('Cancel the deletion')}
|
||||
color="secondary"
|
||||
fullWidth
|
||||
onClick={() => onClose()}
|
||||
onClick={handleClose}
|
||||
onKeyDown={handleCloseKeyDown}
|
||||
>
|
||||
{t('Cancel')}
|
||||
</Button>
|
||||
@@ -78,11 +109,8 @@ export const ModalRemoveDoc = ({
|
||||
aria-label={t('Delete document')}
|
||||
color="danger"
|
||||
fullWidth
|
||||
onClick={() =>
|
||||
removeDoc({
|
||||
docId: doc.id,
|
||||
})
|
||||
}
|
||||
onClick={handleDelete}
|
||||
onKeyDown={handleDeleteKeyDown}
|
||||
>
|
||||
{t('Delete')}
|
||||
</Button>
|
||||
@@ -108,7 +136,8 @@ export const ModalRemoveDoc = ({
|
||||
</Text>
|
||||
<ButtonCloseModal
|
||||
aria-label={t('Close the delete modal')}
|
||||
onClick={() => onClose()}
|
||||
onClick={handleClose}
|
||||
onKeyDown={handleCloseKeyDown}
|
||||
/>
|
||||
</Box>
|
||||
}
|
||||
|
||||
+13
-2
@@ -1,4 +1,5 @@
|
||||
import { useState } from 'react';
|
||||
import { css } from 'styled-components';
|
||||
|
||||
import { BoxButton, Text } from '@/components';
|
||||
import { useCunninghamTheme } from '@/cunningham';
|
||||
@@ -40,7 +41,6 @@ export const Heading = ({
|
||||
<BoxButton
|
||||
id={`heading-${headingId}`}
|
||||
$width="100%"
|
||||
key={headingId}
|
||||
onMouseOver={() => setIsHover(true)}
|
||||
onMouseLeave={() => setIsHover(false)}
|
||||
onClick={() => {
|
||||
@@ -59,8 +59,19 @@ export const Heading = ({
|
||||
}}
|
||||
$radius="4px"
|
||||
$background={isActive ? `${colorsTokens['greyscale-100']}` : 'none'}
|
||||
$css="text-align: left;"
|
||||
$css={css`
|
||||
text-align: left;
|
||||
&:focus-visible {
|
||||
/* Scoped focus style: same footprint as hover, with theme shadow */
|
||||
outline: none;
|
||||
box-shadow: 0 0 0 2px ${colorsTokens['primary-400']};
|
||||
border-radius: 4px;
|
||||
}
|
||||
`}
|
||||
className="--docs--table-content-heading"
|
||||
aria-label={text}
|
||||
aria-selected={isHighlight}
|
||||
aria-current={isHighlight ? 'true' : undefined}
|
||||
>
|
||||
<Text
|
||||
$width="100%"
|
||||
|
||||
+63
-17
@@ -12,7 +12,7 @@ import { Heading } from './Heading';
|
||||
export const TableContent = () => {
|
||||
const { headings } = useHeadingStore();
|
||||
const { editor } = useEditorStore();
|
||||
const { spacingsTokens } = useCunninghamTheme();
|
||||
const { spacingsTokens, colorsTokens } = useCunninghamTheme();
|
||||
|
||||
const [headingIdHighlight, setHeadingIdHighlight] = useState<string>();
|
||||
|
||||
@@ -99,33 +99,62 @@ export const TableContent = () => {
|
||||
|
||||
return (
|
||||
<Box
|
||||
as="nav"
|
||||
id="summaryContainer"
|
||||
$width={!isHover ? '40px' : '200px'}
|
||||
$height={!isHover ? '40px' : 'auto'}
|
||||
$maxHeight="calc(50vh - 60px)"
|
||||
$zIndex={1000}
|
||||
$align="center"
|
||||
$padding="xs"
|
||||
$padding={isHover ? 'xs' : '0'}
|
||||
$justify="center"
|
||||
$position="relative"
|
||||
aria-label={t('Summary')}
|
||||
$css={css`
|
||||
border: 1px solid #ccc;
|
||||
border: 1px solid ${colorsTokens['greyscale-300']};
|
||||
overflow: hidden;
|
||||
border-radius: var(--c--theme--spacings--3xs);
|
||||
background: var(--c--theme--colors--greyscale-000);
|
||||
border-radius: ${spacingsTokens['3xs']};
|
||||
background: ${colorsTokens['greyscale-000']};
|
||||
${isHover &&
|
||||
css`
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: flex-start;
|
||||
align-items: flex-start;
|
||||
gap: var(--c--theme--spacings--2xs);
|
||||
gap: ${spacingsTokens['2xs']};
|
||||
`}
|
||||
`}
|
||||
className="--docs--table-content"
|
||||
>
|
||||
{!isHover && (
|
||||
<BoxButton onClick={onOpen} $justify="center" $align="center">
|
||||
<Icon iconName="list" $theme="primary" $variation="800" />
|
||||
<BoxButton
|
||||
onClick={onOpen}
|
||||
$width="100%"
|
||||
$height="100%"
|
||||
$justify="center"
|
||||
$align="center"
|
||||
aria-label={t('Summary')}
|
||||
aria-expanded={isHover}
|
||||
aria-controls="toc-list"
|
||||
$css={css`
|
||||
&:hover {
|
||||
background: ${colorsTokens['primary-100']};
|
||||
}
|
||||
&:focus-visible {
|
||||
outline: none;
|
||||
box-shadow: 0 0 0 4px ${colorsTokens['primary-400']};
|
||||
background: ${colorsTokens['primary-100']};
|
||||
width: 90%;
|
||||
height: 90%;
|
||||
}
|
||||
`}
|
||||
>
|
||||
<Icon
|
||||
iconName="list"
|
||||
$theme="primary"
|
||||
$variation="800"
|
||||
variant="symbols-outlined"
|
||||
/>
|
||||
</BoxButton>
|
||||
)}
|
||||
{isHover && (
|
||||
@@ -134,10 +163,11 @@ export const TableContent = () => {
|
||||
$overflow="hidden"
|
||||
$css={css`
|
||||
user-select: none;
|
||||
padding: ${spacingsTokens['4xs']};
|
||||
`}
|
||||
>
|
||||
<Box
|
||||
$margin={{ bottom: '10px' }}
|
||||
$margin={{ bottom: spacingsTokens.xs }}
|
||||
$direction="row"
|
||||
$justify="space-between"
|
||||
$align="center"
|
||||
@@ -149,30 +179,46 @@ export const TableContent = () => {
|
||||
onClick={onClose}
|
||||
$justify="center"
|
||||
$align="center"
|
||||
aria-label={t('Summary')}
|
||||
aria-expanded={isHover}
|
||||
aria-controls="toc-list"
|
||||
$css={css`
|
||||
transition: none !important;
|
||||
transform: rotate(180deg);
|
||||
&:focus-visible {
|
||||
outline: none;
|
||||
box-shadow: 0 0 0 2px ${colorsTokens['primary-400']};
|
||||
border-radius: 4px;
|
||||
}
|
||||
`}
|
||||
>
|
||||
<Icon iconName="menu_open" $theme="primary" $variation="800" />
|
||||
</BoxButton>
|
||||
</Box>
|
||||
<Box
|
||||
as="ul"
|
||||
id="toc-list"
|
||||
role="list"
|
||||
$gap={spacingsTokens['3xs']}
|
||||
$css={css`
|
||||
overflow-y: auto;
|
||||
list-style: none;
|
||||
padding: ${spacingsTokens['3xs']};
|
||||
margin: 0;
|
||||
`}
|
||||
>
|
||||
{headings?.map(
|
||||
(heading) =>
|
||||
heading.contentText && (
|
||||
<Heading
|
||||
editor={editor}
|
||||
headingId={heading.id}
|
||||
level={heading.props.level}
|
||||
text={heading.contentText}
|
||||
key={heading.id}
|
||||
isHighlight={headingIdHighlight === heading.id}
|
||||
/>
|
||||
<Box as="li" role="listitem" key={heading.id}>
|
||||
<Heading
|
||||
editor={editor}
|
||||
headingId={heading.id}
|
||||
level={heading.props.level}
|
||||
text={heading.contentText}
|
||||
isHighlight={headingIdHighlight === heading.id}
|
||||
/>
|
||||
</Box>
|
||||
),
|
||||
)}
|
||||
</Box>
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
import { useMutation, useQueryClient } from '@tanstack/react-query';
|
||||
|
||||
import { APIError, errorCauses, fetchAPI } from '@/api';
|
||||
|
||||
import { Doc, KEY_LIST_DOC } from '../../doc-management';
|
||||
|
||||
export const importDoc = async (file: File): Promise<Doc> => {
|
||||
const form = new FormData();
|
||||
form.append('file', file);
|
||||
|
||||
const response = await fetchAPI(`documents/`, {
|
||||
method: 'POST',
|
||||
body: form,
|
||||
withoutContentType: true,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new APIError('Failed to import the doc', await errorCauses(response));
|
||||
}
|
||||
|
||||
return response.json() as Promise<Doc>;
|
||||
};
|
||||
|
||||
interface ImportDocProps {
|
||||
onSuccess?: (data: Doc) => void;
|
||||
onError?: (error: APIError) => void;
|
||||
}
|
||||
|
||||
export function useImportDoc({ onSuccess, onError }: ImportDocProps) {
|
||||
const queryClient = useQueryClient();
|
||||
return useMutation<Doc, APIError, File>({
|
||||
mutationFn: importDoc,
|
||||
onSuccess: (data) => {
|
||||
void queryClient.resetQueries({
|
||||
queryKey: [KEY_LIST_DOC],
|
||||
});
|
||||
onSuccess?.(data);
|
||||
},
|
||||
onError: (error) => {
|
||||
onError?.(error);
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Button } from '@openfun/cunningham-react';
|
||||
import { useMemo, useRef } from 'react';
|
||||
import { useMemo } from 'react';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { InView } from 'react-intersection-observer';
|
||||
import { css } from 'styled-components';
|
||||
@@ -9,7 +9,6 @@ import { DocDefaultFilter, useInfiniteDocs } from '@/docs/doc-management';
|
||||
import { useResponsiveStore } from '@/stores';
|
||||
|
||||
import { useInfiniteDocsTrashbin } from '../api';
|
||||
import { useImportDoc } from '../api/useImportDoc';
|
||||
import { useResponsiveDocGrid } from '../hooks/useResponsiveDocGrid';
|
||||
|
||||
import {
|
||||
@@ -28,7 +27,6 @@ export const DocsGrid = ({
|
||||
|
||||
const { isDesktop } = useResponsiveStore();
|
||||
const { flexLeft, flexRight } = useResponsiveDocGrid();
|
||||
const importInputRef = useRef<HTMLInputElement>(null);
|
||||
|
||||
const {
|
||||
data,
|
||||
@@ -77,33 +75,6 @@ export const DocsGrid = ({
|
||||
title = t('All docs');
|
||||
}
|
||||
|
||||
const resetImportInput = () => {
|
||||
if (!importInputRef.current) {
|
||||
return;
|
||||
}
|
||||
|
||||
importInputRef.current.value = '';
|
||||
};
|
||||
|
||||
const { mutate: importDoc } = useImportDoc({
|
||||
onSuccess: resetImportInput,
|
||||
onError: resetImportInput,
|
||||
});
|
||||
|
||||
const handleImport = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
console.log(event);
|
||||
|
||||
if (!event.target.files || event.target.files.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const file = event.target.files[0];
|
||||
|
||||
console.log(file);
|
||||
|
||||
importDoc(file);
|
||||
};
|
||||
|
||||
return (
|
||||
<Box
|
||||
$position="relative"
|
||||
@@ -136,14 +107,6 @@ export const DocsGrid = ({
|
||||
{title}
|
||||
</Text>
|
||||
|
||||
<input
|
||||
type="file"
|
||||
name="doc"
|
||||
accept=".md,.docx"
|
||||
onChange={handleImport}
|
||||
ref={importInputRef}
|
||||
></input>
|
||||
|
||||
{!hasDocs && !loading && (
|
||||
<Box $padding={{ vertical: 'sm' }} $align="center" $justify="center">
|
||||
<Text $size="sm" $variation="600" $weight="700">
|
||||
@@ -189,23 +152,23 @@ export const DocsGrid = ({
|
||||
<DocGridContentList docs={docs} />
|
||||
)}
|
||||
</Box>
|
||||
{hasNextPage && !loading && (
|
||||
<InView
|
||||
data-testid="infinite-scroll-trigger"
|
||||
as="div"
|
||||
onChange={loadMore}
|
||||
>
|
||||
{!isFetching && hasNextPage && (
|
||||
<Button
|
||||
onClick={() => void fetchNextPage()}
|
||||
color="primary-text"
|
||||
>
|
||||
{t('More docs')}
|
||||
</Button>
|
||||
)}
|
||||
</InView>
|
||||
)}
|
||||
</Box>
|
||||
{hasNextPage && !loading && (
|
||||
<InView
|
||||
data-testid="infinite-scroll-trigger"
|
||||
as="div"
|
||||
onChange={loadMore}
|
||||
>
|
||||
{!isFetching && hasNextPage && (
|
||||
<Button
|
||||
onClick={() => void fetchNextPage()}
|
||||
color="primary-text"
|
||||
>
|
||||
{t('More docs')}
|
||||
</Button>
|
||||
)}
|
||||
</InView>
|
||||
)}
|
||||
</Box>
|
||||
)}
|
||||
</Card>
|
||||
|
||||
@@ -19,7 +19,7 @@ export const Draggable = <T,>(props: DraggableProps<T>) => {
|
||||
{...attributes}
|
||||
data-testid={`draggable-doc-${props.id}`}
|
||||
className="--docs--grid-draggable"
|
||||
role="presentation"
|
||||
role="none"
|
||||
>
|
||||
{props.children}
|
||||
</div>
|
||||
|
||||
@@ -35,7 +35,7 @@ export const Droppable = ({
|
||||
<Box
|
||||
ref={setNodeRef}
|
||||
data-testid={`droppable-doc-${id}`}
|
||||
role="presentation"
|
||||
role="none"
|
||||
$css={css`
|
||||
border-radius: 4px;
|
||||
background-color: ${enableHover
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export * from './useKeyboardAction';
|
||||
@@ -0,0 +1,22 @@
|
||||
import { KeyboardEvent, useCallback } from 'react';
|
||||
|
||||
type KeyboardActionCallback = () => void | Promise<unknown>;
|
||||
type KeyboardActionHandler = (event: KeyboardEvent<HTMLElement>) => void;
|
||||
|
||||
/**
|
||||
* Hook to create keyboard handlers that trigger the provided callback
|
||||
* when the user presses Enter or Space.
|
||||
*/
|
||||
export const useKeyboardAction = () => {
|
||||
return useCallback(
|
||||
(callback: KeyboardActionCallback): KeyboardActionHandler =>
|
||||
(event: KeyboardEvent<HTMLElement>) => {
|
||||
if (event.key === 'Enter' || event.key === ' ') {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
void callback();
|
||||
}
|
||||
},
|
||||
[],
|
||||
);
|
||||
};
|
||||
@@ -77,7 +77,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "Pellgargañ",
|
||||
"Download anyway": "Pellgargañ memestra",
|
||||
"Download your document in a .docx or .pdf format.": "Pellgargañ ho restr dindan ur stumm .docx pe .pdf.",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Pellgargañ ho restr dindan ur stumm .docx, .odt pe .pdf.",
|
||||
"Duplicate": "Eilañ",
|
||||
"Editing": "Oc'h aozañ",
|
||||
"Editor": "Embanner",
|
||||
@@ -296,7 +296,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "Herunterladen",
|
||||
"Download anyway": "Trotzdem herunterladen",
|
||||
"Download your document in a .docx or .pdf format.": "Ihr Dokument als .docx- oder .pdf-Datei herunterladen.",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Ihr Dokument als .docx-, .odt- oder .pdf-Datei herunterladen.",
|
||||
"Duplicate": "Duplizieren",
|
||||
"Editing": "Bearbeiten",
|
||||
"Editor": "Mitbearbeiter",
|
||||
@@ -494,7 +494,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "Descargar",
|
||||
"Download anyway": "Descargar de todos modos",
|
||||
"Download your document in a .docx or .pdf format.": "Descargue su documento en formato .docx o .pdf.",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Descargue su documento en formato .docx, .odt o .pdf.",
|
||||
"Editor": "Editor",
|
||||
"Editor unavailable": "Editor no disponible",
|
||||
"Emojify": "Emojizar",
|
||||
@@ -698,7 +698,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "Télécharger",
|
||||
"Download anyway": "Télécharger malgré tout",
|
||||
"Download your document in a .docx or .pdf format.": "Téléchargez votre document au format .docx ou .pdf.",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Téléchargez votre document au format .docx, .odt ou .pdf.",
|
||||
"Drag and drop status": "État du glisser-déposer",
|
||||
"Duplicate": "Dupliquer",
|
||||
"Edit document emoji": "Modifier l'émoticône du document",
|
||||
@@ -930,7 +930,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "Scarica",
|
||||
"Download anyway": "Scarica comunque",
|
||||
"Download your document in a .docx or .pdf format.": "Scarica il tuo documento in formato .docx o .pdf",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Scarica il tuo documento in formato .docx, .odt o .pdf",
|
||||
"Editor": "Editor",
|
||||
"Editor unavailable": "Editor non disponibile",
|
||||
"Emojify": "Emojify",
|
||||
@@ -1117,7 +1117,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "Download",
|
||||
"Download anyway": "Download alsnog",
|
||||
"Download your document in a .docx or .pdf format.": "Download jouw document in .docx of .pdf formaat.",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Download jouw document in .docx, .odt of .pdf formaat.",
|
||||
"Drag and drop status": "Drag & drop status",
|
||||
"Duplicate": "Dupliceer",
|
||||
"Edit document emoji": "Bewerk document emoji",
|
||||
@@ -1394,7 +1394,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "Загрузить",
|
||||
"Download anyway": "Всё равно загрузить",
|
||||
"Download your document in a .docx or .pdf format.": "Загрузите свой документ в формате .docx или .pdf.",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Загрузите свой документ в формате .docx, .odt или .pdf.",
|
||||
"Drag and drop status": "Состояние перетаскивания",
|
||||
"Duplicate": "Дублировать",
|
||||
"Editing": "Редактирование",
|
||||
@@ -1642,7 +1642,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "İndir",
|
||||
"Download anyway": "Yine de indir",
|
||||
"Download your document in a .docx or .pdf format.": "Belgenizi .docx veya .pdf formatında indirin.",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Belgenizi .docx, .odt veya .pdf formatında indirin.",
|
||||
"Editor": "Editör",
|
||||
"Editor unavailable": "Editör mevcut değil",
|
||||
"Emojify": "Emojileştir",
|
||||
@@ -1774,7 +1774,7 @@
|
||||
"Docx": "Docx",
|
||||
"Download": "Завантажити",
|
||||
"Download anyway": "Все одно завантажити",
|
||||
"Download your document in a .docx or .pdf format.": "Завантажте ваш документ у форматі .docx або .pdf.",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "Завантажте ваш документ у форматі .docx, .odt або .pdf.",
|
||||
"Drag and drop status": "Стан перетягування",
|
||||
"Duplicate": "Дублювати",
|
||||
"Editing": "Редагування",
|
||||
@@ -2018,7 +2018,7 @@
|
||||
"Docx": "Doc",
|
||||
"Download": "下载",
|
||||
"Download anyway": "仍要下载",
|
||||
"Download your document in a .docx or .pdf format.": "以doc或者pdf格式下载。",
|
||||
"Download your document in a .docx, .odt or .pdf format.": "以doc, odt或者pdf格式下载。",
|
||||
"Editing": "正在编辑",
|
||||
"Editor": "编辑者",
|
||||
"Editor unavailable": "编辑功能不可用",
|
||||
|
||||
@@ -69,7 +69,7 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer wrong-api-key`)
|
||||
.set('authorization', 'wrong-api-key')
|
||||
.set('content-type', 'application/json');
|
||||
|
||||
expect(response.status).toBe(401);
|
||||
@@ -99,7 +99,7 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'application/json');
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
@@ -114,7 +114,7 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'application/json')
|
||||
.send('');
|
||||
|
||||
@@ -129,10 +129,9 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'image/png')
|
||||
.send('randomdata');
|
||||
|
||||
expect(response.status).toBe(415);
|
||||
expect(response.body).toStrictEqual({ error: 'Unsupported Content-Type' });
|
||||
});
|
||||
@@ -142,73 +141,38 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'text/markdown')
|
||||
.set('accept', 'image/png')
|
||||
.send('# Header');
|
||||
|
||||
expect(response.status).toBe(406);
|
||||
expect(response.body).toStrictEqual({ error: 'Unsupported format' });
|
||||
});
|
||||
|
||||
test('POST /api/convert BlockNote to Markdown', async () => {
|
||||
const app = initApp();
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('content-type', 'application/vnd.blocknote+json')
|
||||
.set('accept', 'text/markdown')
|
||||
.send(expectedBlocks);
|
||||
test.each([[apiKey], [`Bearer ${apiKey}`]])(
|
||||
'POST /api/convert with correct content with Authorization: %s',
|
||||
async (authHeader) => {
|
||||
const app = initApp();
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.header['content-type']).toBe(
|
||||
'text/markdown; charset=utf-8',
|
||||
);
|
||||
expect(typeof response.text).toBe('string');
|
||||
expect(response.text.trim()).toBe(expectedMarkdown);
|
||||
});
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('Origin', origin)
|
||||
.set('Authorization', authHeader)
|
||||
.set('content-type', 'text/markdown')
|
||||
.set('accept', 'application/vnd.yjs.doc')
|
||||
.send(expectedMarkdown);
|
||||
|
||||
test('POST /api/convert BlockNote to Yjs', async () => {
|
||||
const app = initApp();
|
||||
const editor = ServerBlockNoteEditor.create();
|
||||
const blocks = await editor.tryParseMarkdownToBlocks(expectedMarkdown);
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('content-type', 'application/vnd.blocknote+json')
|
||||
.set('accept', 'application/vnd.yjs.doc')
|
||||
.send(blocks)
|
||||
.responseType('blob');
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.body).toBeInstanceOf(Buffer);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.header['content-type']).toBe('application/vnd.yjs.doc');
|
||||
const editor = ServerBlockNoteEditor.create();
|
||||
const doc = new Y.Doc();
|
||||
Y.applyUpdate(doc, response.body);
|
||||
const blocks = editor.yDocToBlocks(doc, 'document-store');
|
||||
|
||||
// Decode the Yjs response and verify it contains the correct blocks
|
||||
const responseBuffer = Buffer.from(response.body as Buffer);
|
||||
const ydoc = new Y.Doc();
|
||||
Y.applyUpdate(ydoc, responseBuffer);
|
||||
const decodedBlocks = editor.yDocToBlocks(ydoc, 'document-store');
|
||||
|
||||
expect(decodedBlocks).toStrictEqual(expectedBlocks);
|
||||
});
|
||||
|
||||
test('POST /api/convert BlockNote to HTML', async () => {
|
||||
const app = initApp();
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('content-type', 'application/vnd.blocknote+json')
|
||||
.set('accept', 'text/html')
|
||||
.send(expectedBlocks);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.header['content-type']).toBe('text/html; charset=utf-8');
|
||||
expect(typeof response.text).toBe('string');
|
||||
expect(response.text).toBe(expectedHTML);
|
||||
});
|
||||
expect(blocks).toStrictEqual(expectedBlocks);
|
||||
},
|
||||
);
|
||||
|
||||
test('POST /api/convert Yjs to HTML', async () => {
|
||||
const app = initApp();
|
||||
@@ -219,11 +183,10 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'application/vnd.yjs.doc')
|
||||
.set('accept', 'text/html')
|
||||
.send(Buffer.from(yjsUpdate));
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.header['content-type']).toBe('text/html; charset=utf-8');
|
||||
expect(typeof response.text).toBe('string');
|
||||
@@ -239,11 +202,10 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'application/vnd.yjs.doc')
|
||||
.set('accept', 'text/markdown')
|
||||
.send(Buffer.from(yjsUpdate));
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.header['content-type']).toBe(
|
||||
'text/markdown; charset=utf-8',
|
||||
@@ -261,16 +223,15 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'application/vnd.yjs.doc')
|
||||
.set('accept', 'application/json')
|
||||
.send(Buffer.from(yjsUpdate));
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.header['content-type']).toBe(
|
||||
'application/json; charset=utf-8',
|
||||
);
|
||||
expect(response.body).toBeInstanceOf(Array);
|
||||
expect(Array.isArray(response.body)).toBe(true);
|
||||
expect(response.body).toStrictEqual(expectedBlocks);
|
||||
});
|
||||
|
||||
@@ -279,16 +240,15 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'text/markdown')
|
||||
.set('accept', 'application/json')
|
||||
.send(expectedMarkdown);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.header['content-type']).toBe(
|
||||
'application/json; charset=utf-8',
|
||||
);
|
||||
expect(response.body).toBeInstanceOf(Array);
|
||||
expect(Array.isArray(response.body)).toBe(true);
|
||||
expect(response.body).toStrictEqual(expectedBlocks);
|
||||
});
|
||||
|
||||
@@ -297,12 +257,11 @@ describe('Server Tests', () => {
|
||||
const response = await request(app)
|
||||
.post('/api/convert')
|
||||
.set('origin', origin)
|
||||
.set('authorization', `Bearer ${apiKey}`)
|
||||
.set('authorization', apiKey)
|
||||
.set('content-type', 'application/vnd.yjs.doc')
|
||||
.set('accept', 'application/json')
|
||||
.send(Buffer.from('notvalidyjs'));
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
expect(response.body).toStrictEqual({ error: 'Invalid content' });
|
||||
expect(response.body).toStrictEqual({ error: 'Invalid Yjs content' });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -14,115 +14,27 @@ interface ErrorResponse {
|
||||
error: string;
|
||||
}
|
||||
|
||||
type ConversionResponseBody = Uint8Array | string | object | ErrorResponse;
|
||||
|
||||
interface InputReader {
|
||||
supportedContentTypes: string[];
|
||||
read(data: Buffer): Promise<PartialBlock[]>;
|
||||
}
|
||||
|
||||
interface OutputWriter {
|
||||
supportedContentTypes: string[];
|
||||
write(blocks: PartialBlock[]): Promise<ConversionResponseBody>;
|
||||
}
|
||||
|
||||
const editor = ServerBlockNoteEditor.create<
|
||||
DefaultBlockSchema,
|
||||
DefaultInlineContentSchema,
|
||||
DefaultStyleSchema
|
||||
>();
|
||||
|
||||
const ContentTypes = {
|
||||
XMarkdown: 'text/x-markdown',
|
||||
Markdown: 'text/markdown',
|
||||
YJS: 'application/vnd.yjs.doc',
|
||||
FormUrlEncoded: 'application/x-www-form-urlencoded',
|
||||
OctetStream: 'application/octet-stream',
|
||||
HTML: 'text/html',
|
||||
BlockNote: 'application/vnd.blocknote+json',
|
||||
JSON: 'application/json',
|
||||
} as const;
|
||||
|
||||
const createYDocument = (blocks: PartialBlock[]) =>
|
||||
editor.blocksToYDoc(blocks, 'document-store');
|
||||
|
||||
const readers: InputReader[] = [
|
||||
{
|
||||
// application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility
|
||||
supportedContentTypes: [
|
||||
ContentTypes.Markdown,
|
||||
ContentTypes.XMarkdown,
|
||||
ContentTypes.FormUrlEncoded,
|
||||
],
|
||||
read: (data) => editor.tryParseMarkdownToBlocks(data.toString()),
|
||||
},
|
||||
{
|
||||
supportedContentTypes: [ContentTypes.YJS, ContentTypes.OctetStream],
|
||||
read: async (data) => {
|
||||
const ydoc = new Y.Doc();
|
||||
Y.applyUpdate(ydoc, data);
|
||||
return editor.yDocToBlocks(ydoc, 'document-store') as PartialBlock[];
|
||||
},
|
||||
},
|
||||
{
|
||||
supportedContentTypes: [ContentTypes.BlockNote],
|
||||
read: async (data) => JSON.parse(data.toString()),
|
||||
},
|
||||
];
|
||||
|
||||
const writers: OutputWriter[] = [
|
||||
{
|
||||
supportedContentTypes: [ContentTypes.BlockNote, ContentTypes.JSON],
|
||||
write: async (blocks) => blocks,
|
||||
},
|
||||
{
|
||||
supportedContentTypes: [ContentTypes.YJS, ContentTypes.OctetStream],
|
||||
write: async (blocks) => Y.encodeStateAsUpdate(createYDocument(blocks)),
|
||||
},
|
||||
{
|
||||
supportedContentTypes: [ContentTypes.Markdown, ContentTypes.XMarkdown],
|
||||
write: (blocks) => editor.blocksToMarkdownLossy(blocks),
|
||||
},
|
||||
{
|
||||
supportedContentTypes: [ContentTypes.HTML],
|
||||
write: (blocks) => editor.blocksToHTMLLossy(blocks),
|
||||
},
|
||||
];
|
||||
|
||||
const normalizeContentType = (value: string) => value.split(';')[0];
|
||||
|
||||
export const convertHandler = async (
|
||||
req: Request<object, Uint8Array | ErrorResponse, Buffer, object>,
|
||||
res: Response<ConversionResponseBody>,
|
||||
res: Response<Uint8Array | string | object | ErrorResponse>,
|
||||
) => {
|
||||
if (!req.body || req.body.length === 0) {
|
||||
res.status(400).json({ error: 'Invalid request: missing content' });
|
||||
return;
|
||||
}
|
||||
|
||||
const contentType = normalizeContentType(
|
||||
req.header('content-type') || ContentTypes.Markdown,
|
||||
);
|
||||
|
||||
const reader = readers.find((reader) =>
|
||||
reader.supportedContentTypes.includes(contentType),
|
||||
);
|
||||
|
||||
if (!reader) {
|
||||
res.status(415).json({ error: 'Unsupported Content-Type' });
|
||||
return;
|
||||
}
|
||||
|
||||
const accept = normalizeContentType(req.header('accept') || ContentTypes.YJS);
|
||||
|
||||
const writer = writers.find((writer) =>
|
||||
writer.supportedContentTypes.includes(accept),
|
||||
);
|
||||
|
||||
if (!writer) {
|
||||
res.status(406).json({ error: 'Unsupported format' });
|
||||
return;
|
||||
}
|
||||
const contentType = (req.header('content-type') || 'text/markdown').split(
|
||||
';',
|
||||
)[0];
|
||||
const accept = (req.header('accept') || 'application/vnd.yjs.doc').split(
|
||||
';',
|
||||
)[0];
|
||||
|
||||
let blocks:
|
||||
| PartialBlock<
|
||||
@@ -132,23 +44,63 @@ export const convertHandler = async (
|
||||
>[]
|
||||
| null = null;
|
||||
try {
|
||||
try {
|
||||
blocks = await reader.read(req.body);
|
||||
} catch (e) {
|
||||
logger('Invalid content:', e);
|
||||
res.status(400).json({ error: 'Invalid content' });
|
||||
// First, convert from the input format to blocks
|
||||
// application/x-www-form-urlencoded is interpreted as Markdown for backward compatibility
|
||||
if (
|
||||
contentType === 'text/markdown' ||
|
||||
contentType === 'application/x-www-form-urlencoded'
|
||||
) {
|
||||
blocks = await editor.tryParseMarkdownToBlocks(req.body.toString());
|
||||
} else if (
|
||||
contentType === 'application/vnd.yjs.doc' ||
|
||||
contentType === 'application/octet-stream'
|
||||
) {
|
||||
try {
|
||||
const ydoc = new Y.Doc();
|
||||
Y.applyUpdate(ydoc, req.body);
|
||||
blocks = editor.yDocToBlocks(ydoc, 'document-store') as PartialBlock[];
|
||||
} catch (e) {
|
||||
logger('Invalid Yjs content:', e);
|
||||
res.status(400).json({ error: 'Invalid Yjs content' });
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
res.status(415).json({ error: 'Unsupported Content-Type' });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!blocks || blocks.length === 0) {
|
||||
res.status(500).json({ error: 'No valid blocks were generated' });
|
||||
return;
|
||||
}
|
||||
|
||||
res
|
||||
.status(200)
|
||||
.setHeader('content-type', accept)
|
||||
.send(await writer.write(blocks));
|
||||
// Then, convert from blocks to the output format
|
||||
if (accept === 'application/json') {
|
||||
res.status(200).json(blocks);
|
||||
} else {
|
||||
const yDocument = editor.blocksToYDoc(blocks, 'document-store');
|
||||
|
||||
if (
|
||||
accept === 'application/vnd.yjs.doc' ||
|
||||
accept === 'application/octet-stream'
|
||||
) {
|
||||
res
|
||||
.status(200)
|
||||
.setHeader('content-type', 'application/octet-stream')
|
||||
.send(Y.encodeStateAsUpdate(yDocument));
|
||||
} else if (accept === 'text/markdown') {
|
||||
res
|
||||
.status(200)
|
||||
.setHeader('content-type', 'text/markdown')
|
||||
.send(await editor.blocksToMarkdownLossy(blocks));
|
||||
} else if (accept === 'text/html') {
|
||||
res
|
||||
.status(200)
|
||||
.setHeader('content-type', 'text/html')
|
||||
.send(await editor.blocksToHTMLLossy(blocks));
|
||||
} else {
|
||||
res.status(406).json({ error: 'Unsupported format' });
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
logger('conversion failed:', e);
|
||||
res.status(500).json({ error: 'An error occurred' });
|
||||
|
||||
@@ -1192,6 +1192,17 @@
|
||||
prosemirror-view "^1.41.2"
|
||||
react-icons "^5.2.1"
|
||||
|
||||
"@blocknote/xl-odt-exporter@0.41.1":
|
||||
version "0.41.1"
|
||||
resolved "https://registry.yarnpkg.com/@blocknote/xl-odt-exporter/-/xl-odt-exporter-0.41.1.tgz#55be888d7b6158a6e352aab414cee12e1dcf4326"
|
||||
integrity sha512-VAQC8isRoioK097yuFX0p6dIrwp/GyWInd4hDkux3gsGTMqdXRiLV42symC6+qEseukz+IbGqWvWGsgAplwkZQ==
|
||||
dependencies:
|
||||
"@blocknote/core" "0.41.1"
|
||||
"@blocknote/xl-multi-column" "0.41.1"
|
||||
"@zip.js/zip.js" "^2.7.57"
|
||||
buffer "^6.0.3"
|
||||
image-meta "^0.2.1"
|
||||
|
||||
"@blocknote/xl-pdf-exporter@0.41.1":
|
||||
version "0.41.1"
|
||||
resolved "https://registry.yarnpkg.com/@blocknote/xl-pdf-exporter/-/xl-pdf-exporter-0.41.1.tgz#b55c7e8c6a21ae069a42671b1391eab9d4119195"
|
||||
@@ -6253,6 +6264,11 @@
|
||||
resolved "https://registry.yarnpkg.com/@xtuc/long/-/long-4.2.2.tgz#d291c6a4e97989b5c61d9acf396ae4fe133a718d"
|
||||
integrity sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==
|
||||
|
||||
"@zip.js/zip.js@^2.7.57":
|
||||
version "2.8.10"
|
||||
resolved "https://registry.yarnpkg.com/@zip.js/zip.js/-/zip.js-2.8.10.tgz#98a0cc7fdef9d6e227236271af412db02b18a5b2"
|
||||
integrity sha512-WVywWK8HSttmFFYSih7lUjjaV4zGzMxy992y0tHrZY4Wf9x/uNBA/XJ50RvfGjuuJKti4yueEHA2ol2pOq6VDg==
|
||||
|
||||
abs-svg-path@^0.1.1:
|
||||
version "0.1.1"
|
||||
resolved "https://registry.yarnpkg.com/abs-svg-path/-/abs-svg-path-0.1.1.tgz#df601c8e8d2ba10d4a76d625e236a9a39c2723bf"
|
||||
|
||||
@@ -166,14 +166,6 @@ Requires top level scope
|
||||
{{ include "impress.fullname" . }}-y-provider
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Full name for the docSpecApi
|
||||
|
||||
Requires top level scope
|
||||
*/}}
|
||||
{{- define "impress.docSpecApi.fullname" -}}
|
||||
{{ include "impress.fullname" . }}-docspec-api
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Full name for the Celery Worker
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
{{- $envVars := include "impress.common.env" (list . .Values.docSpecApi) -}}
|
||||
{{- $fullName := include "impress.docSpecApi.fullname" . -}}
|
||||
{{- $component := "docspec" -}}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ $fullName }}
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
annotations:
|
||||
{{- with .Values.backend.dpAnnotations }}
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "impress.common.labels" (list . $component) | nindent 4 }}
|
||||
spec:
|
||||
replicas: {{ .Values.docSpecApi.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
{{- with .Values.docSpecApi.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "impress.common.selectorLabels" (list . $component) | nindent 8 }}
|
||||
spec:
|
||||
{{- if $.Values.image.credentials }}
|
||||
imagePullSecrets:
|
||||
- name: {{ include "impress.secret.dockerconfigjson.name" (dict "fullname" (include "impress.fullname" .) "imageCredentials" $.Values.image.credentials) }}
|
||||
{{- end}}
|
||||
{{- if .Values.docSpecApi.serviceAccountName }}
|
||||
serviceAccountName: {{ .Values.docSpecApi.serviceAccountName }}
|
||||
{{- end }}
|
||||
shareProcessNamespace: {{ .Values.docSpecApi.shareProcessNamespace }}
|
||||
containers:
|
||||
{{- with .Values.docSpecApi.sidecars }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
- name: {{ .Chart.Name }}
|
||||
image: "{{ (.Values.docSpecApi.image | default dict).repository | default .Values.image.repository }}:{{ (.Values.docSpecApi.image | default dict).tag | default .Values.image.tag }}"
|
||||
imagePullPolicy: {{ (.Values.docSpecApi.image | default dict).pullPolicy | default .Values.image.pullPolicy }}
|
||||
{{- with .Values.yPrdocspecovider.command }}
|
||||
command:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.docSpecApi.args }}
|
||||
args:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
env:
|
||||
{{- if $envVars}}
|
||||
{{- $envVars | indent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.docSpecApi.securityContext }}
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: {{ .Values.docSpecApi.service.targetPort }}
|
||||
protocol: TCP
|
||||
{{- if .Values.docSpecApi.probes.liveness }}
|
||||
livenessProbe:
|
||||
{{- include "impress.probes.abstract" (merge .Values.docSpecApi.probes.liveness (dict "targetPort" .Values.docSpecApi.service.targetPort )) | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.docSpecApi.probes.readiness }}
|
||||
readinessProbe:
|
||||
{{- include "impress.probes.abstract" (merge .Values.docSpecApi.probes.readiness (dict "targetPort" .Values.docSpecApi.service.targetPort )) | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- if .Values.docSpecApi.probes.startup }}
|
||||
startupProbe:
|
||||
{{- include "impress.probes.abstract" (merge .Values.docSpecApi.probes.startup (dict "targetPort" .Values.docSpecApi.service.targetPort )) | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.docSpecApi.resources }}
|
||||
resources:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
{{- range $index, $value := .Values.mountFiles }}
|
||||
- name: "files-{{ $index }}"
|
||||
mountPath: {{ $value.path }}
|
||||
subPath: content
|
||||
{{- end }}
|
||||
{{- range $name, $volume := .Values.docSpecApi.persistence }}
|
||||
- name: "{{ $name }}"
|
||||
mountPath: "{{ $volume.mountPath }}"
|
||||
{{- end }}
|
||||
{{- range .Values.docSpecApi.extraVolumeMounts }}
|
||||
- name: {{ .name }}
|
||||
mountPath: {{ .mountPath }}
|
||||
subPath: {{ .subPath | default "" }}
|
||||
readOnly: {{ .readOnly }}
|
||||
{{- end }}
|
||||
{{- with .Values.docSpecApi.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.docSpecApi.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.docSpecApi.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
{{- range $index, $value := .Values.mountFiles }}
|
||||
- name: "files-{{ $index }}"
|
||||
configMap:
|
||||
name: "{{ include "impress.fullname" $ }}-files-{{ $index }}"
|
||||
{{- end }}
|
||||
{{- range $name, $volume := .Values.docSpecApi.persistence }}
|
||||
- name: "{{ $name }}"
|
||||
{{- if eq $volume.type "emptyDir" }}
|
||||
emptyDir: {}
|
||||
{{- else }}
|
||||
persistentVolumeClaim:
|
||||
claimName: "{{ $fullName }}-{{ $name }}"
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- range .Values.docSpecApi.extraVolumes }}
|
||||
- name: {{ .name }}
|
||||
{{- if .existingClaim }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ .existingClaim }}
|
||||
{{- else if .hostPath }}
|
||||
hostPath:
|
||||
{{ toYaml .hostPath | nindent 12 }}
|
||||
{{- else if .csi }}
|
||||
csi:
|
||||
{{- toYaml .csi | nindent 12 }}
|
||||
{{- else if .configMap }}
|
||||
configMap:
|
||||
{{- toYaml .configMap | nindent 12 }}
|
||||
{{- else if .emptyDir }}
|
||||
emptyDir:
|
||||
{{- toYaml .emptyDir | nindent 12 }}
|
||||
{{- else }}
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
---
|
||||
{{ if .Values.docSpecApi.pdb.enabled }}
|
||||
apiVersion: policy/v1
|
||||
kind: PodDisruptionBudget
|
||||
metadata:
|
||||
name: {{ $fullName }}
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
spec:
|
||||
maxUnavailable: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "impress.common.selectorLabels" (list . $component) | nindent 6 }}
|
||||
{{ end }}
|
||||
@@ -1,21 +0,0 @@
|
||||
{{- $envVars := include "impress.common.env" (list . .Values.yProvider) -}}
|
||||
{{- $fullName := include "impress.docspec.fullname" . -}}
|
||||
{{- $component := "docspec" -}}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ $fullName }}
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
labels:
|
||||
{{- include "impress.common.labels" (list . $component) | nindent 4 }}
|
||||
annotations:
|
||||
{{- toYaml $.Values.docspec.service.annotations | nindent 4 }}
|
||||
spec:
|
||||
type: {{ .Values.docspec.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.docspec.service.port }}
|
||||
targetPort: {{ .Values.docspec.service.targetPort }}
|
||||
protocol: TCP
|
||||
name: http
|
||||
selector:
|
||||
{{- include "impress.common.selectorLabels" (list . $component) | nindent 4 }}
|
||||
@@ -1,67 +0,0 @@
|
||||
{{- if .Values.ingressDocSpecApi.enabled -}}
|
||||
{{- $fullName := include "impress.fullname" . -}}
|
||||
{{- if and .Values.ingressDocSpecApi.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
|
||||
{{- if not (hasKey .Values.ingressDocSpecApi.annotations "kubernetes.io/ingress.class") }}
|
||||
{{- $_ := set .Values.ingressDocSpecApi.annotations "kubernetes.io/ingress.class" .Values.ingressCollaborationApi.className}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
{{- else -}}
|
||||
apiVersion: extensions/v1beta1
|
||||
{{- end }}
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ $fullName }}-collaboration-api
|
||||
namespace: {{ .Release.Namespace | quote }}
|
||||
labels:
|
||||
{{- include "impress.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingressDocSpecApi.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if and .Values.ingressDocSpecApi.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
|
||||
ingressClassName: {{ .Values.ingressDocSpecApi.className }}
|
||||
{{- end }}
|
||||
{{- if .Values.ingressDocSpecApi.tls.enabled }}
|
||||
tls:
|
||||
{{- if .Values.ingressDocSpecApi.host }}
|
||||
- secretName: {{ .Values.ingressDocSpecApi.tls.secretName | default (printf "%s-tls" $fullName) | quote }}
|
||||
hosts:
|
||||
- {{ .Values.ingressDocSpecApi.host | quote }}
|
||||
{{- end }}
|
||||
{{- range .Values.ingressDocSpecApi.tls.additional }}
|
||||
- hosts:
|
||||
{{- range .hosts }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
secretName: {{ .secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- if .Values.ingressDocSpecApi.host }}
|
||||
- host: {{ .Values.ingressDocSpecApi.host | quote }}
|
||||
http:
|
||||
paths:
|
||||
- path: {{ .Values.ingressDocSpecApi.path | quote }}
|
||||
{{- if semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion }}
|
||||
pathType: ImplementationSpecific
|
||||
{{- end }}
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ include "impress.docspec.fullname" . }}
|
||||
port:
|
||||
number: {{ .Values.docspec.service.port }}
|
||||
{{- else }}
|
||||
serviceName: {{ include "impress.docspec.fullname" . }}
|
||||
servicePort: {{ .Values.docspec.service.port }}
|
||||
{{- end }}
|
||||
{{- with .Values.ingressDocSpecApi.customBackends }}
|
||||
{{- toYaml . | nindent 10 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -661,109 +661,3 @@ yProvider:
|
||||
|
||||
## @param yProvider.serviceAccountName Optional service account name to use for yProvider pods
|
||||
serviceAccountName: null
|
||||
|
||||
|
||||
## @section docSpecApi
|
||||
|
||||
docSpecApi:
|
||||
## @param docSpecApi.image.repository Repository to use to pull impress's docSpecApi container image
|
||||
## @param docSpecApi.image.tag impress's docSpecApi container tag
|
||||
## @param docSpecApi.image.pullPolicy docSpecApi container image pull policy
|
||||
image:
|
||||
repository: ghcr.io/docspecio/api
|
||||
pullPolicy: IfNotPresent
|
||||
tag: "latest"
|
||||
|
||||
## @param docSpecApi.command Override the docSpecApi container command
|
||||
command: []
|
||||
|
||||
## @param docSpecApi.args Override the docSpecApi container args
|
||||
args: []
|
||||
|
||||
## @param docSpecApi.replicas Amount of docSpecApi replicas
|
||||
replicas: 3
|
||||
|
||||
## @param docSpecApi.shareProcessNamespace Enable share process namedocSpecApi between containers
|
||||
shareProcessNamespace: false
|
||||
|
||||
## @param docSpecApi.sidecars Add sidecars containers to docSpecApi deployment
|
||||
sidecars: []
|
||||
|
||||
## @param docSpecApi.securityContext Configure docSpecApi Pod security context
|
||||
securityContext: null
|
||||
|
||||
## @param docSpecApi.envVars Configure docSpecApi container environment variables
|
||||
## @extra docSpecApi.envVars.BY_VALUE Example environment variable by setting value directly
|
||||
## @extra docSpecApi.envVars.FROM_CONFIGMAP.configMapKeyRef.name Name of a ConfigMap when configuring env vars from a ConfigMap
|
||||
## @extra docSpecApi.envVars.FROM_CONFIGMAP.configMapKeyRef.key Key within a ConfigMap when configuring env vars from a ConfigMap
|
||||
## @extra docSpecApi.envVars.FROM_SECRET.secretKeyRef.name Name of a Secret when configuring env vars from a Secret
|
||||
## @extra docSpecApi.envVars.FROM_SECRET.secretKeyRef.key Key within a Secret when configuring env vars from a Secret
|
||||
## @skip docSpecApi.envVars
|
||||
envVars:
|
||||
<<: *commonEnvVars
|
||||
|
||||
## @param docSpecApi.podAnnotations Annotations to add to the docSpecApi Pod
|
||||
podAnnotations: {}
|
||||
|
||||
## @param docSpecApi.dpAnnotations Annotations to add to the docSpecApi Deployment
|
||||
dpAnnotations: {}
|
||||
|
||||
## @param docSpecApi.service.type docSpecApi Service type
|
||||
## @param docSpecApi.service.port docSpecApi Service listening port
|
||||
## @param docSpecApi.service.targetPort docSpecApi container listening port
|
||||
## @param docSpecApi.service.annotations Annotations to add to the docSpecApi Service
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 443
|
||||
targetPort: 4000
|
||||
annotations: {}
|
||||
|
||||
## @extra docSpecApi.probes.liveness.path Configure path for docSpecApi HTTP liveness probe
|
||||
## @extra docSpecApi.probes.liveness.targetPort Configure port for docSpecApi HTTP liveness probe
|
||||
## @extra docSpecApi.probes.liveness.initialDelaySeconds Configure initial delay for docSpecApi liveness probe
|
||||
## @extra docSpecApi.probes.liveness.initialDelaySeconds Configure timeout for docSpecApi liveness probe
|
||||
## @extra docSpecApi.probes.startup.path Configure path for docSpecApi HTTP startup probe
|
||||
## @extra docSpecApi.probes.startup.targetPort Configure port for docSpecApi HTTP startup probe
|
||||
## @extra docSpecApi.probes.startup.initialDelaySeconds Configure initial delay for docSpecApi startup probe
|
||||
## @extra docSpecApi.probes.startup.initialDelaySeconds Configure timeout for docSpecApi startup probe
|
||||
## @extra docSpecApi.probes.readiness.path Configure path for docSpecApi HTTP readiness probe
|
||||
## @extra docSpecApi.probes.readiness.targetPort Configure port for docSpecApi HTTP readiness probe
|
||||
## @extra docSpecApi.probes.readiness.initialDelaySeconds Configure initial delay for docSpecApi readiness probe
|
||||
## @extra docSpecApi.probes.readiness.initialDelaySeconds Configure timeout for docSpecApi readiness probe
|
||||
probes:
|
||||
liveness:
|
||||
## @param docSpecApi.probes.liveness.path
|
||||
## @param docSpecApi.probes.liveness.initialDelaySeconds
|
||||
path: /health
|
||||
initialDelaySeconds: 10
|
||||
|
||||
## @param docSpecApi.resources Resource requirements for the docSpecApi container
|
||||
resources: {}
|
||||
|
||||
## @param docSpecApi.nodeSelector Node selector for the docSpecApi Pod
|
||||
nodeSelector: {}
|
||||
|
||||
## @param docSpecApi.tolerations Tolerations for the docSpecApi Pod
|
||||
tolerations: []
|
||||
|
||||
## @param docSpecApi.affinity Affinity for the docSpecApi Pod
|
||||
affinity: {}
|
||||
|
||||
## @param docSpecApi.persistence Additional volumes to create and mount on the docSpecApi. Used for debugging purposes
|
||||
## @extra docSpecApi.persistence.volume-name.size Size of the additional volume
|
||||
## @extra docSpecApi.persistence.volume-name.type Type of the additional volume, persistentVolumeClaim or emptyDir
|
||||
## @extra docSpecApi.persistence.volume-name.mountPath Path where the volume should be mounted to
|
||||
persistence: {}
|
||||
|
||||
## @param docSpecApi.extraVolumeMounts Additional volumes to mount on the docSpecApi.
|
||||
extraVolumeMounts: []
|
||||
|
||||
## @param docSpecApi.extraVolumes Additional volumes to mount on the docSpecApi.
|
||||
extraVolumes: []
|
||||
|
||||
## @param docSpecApi.pdb.enabled Enable pdb on docSpecApi
|
||||
pdb:
|
||||
enabled: true
|
||||
|
||||
## @param docSpecApi.serviceAccountName Optional service account name to use for docSpecApi pods
|
||||
serviceAccountName: null
|
||||
|
||||
Reference in New Issue
Block a user