Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ef1679968d | |||
| a42c35dda0 | |||
| b2192b9594 | |||
| 75c649be39 | |||
| 03fd1fe50e | |||
| fc803226ac | |||
| fb725edda3 | |||
| 6838b387a2 | |||
| 87f570582f | |||
| 37f56fcc22 | |||
| 19aa3a36bc |
+1
-1
@@ -9,12 +9,12 @@ and this project adheres to
|
|||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- 💄(frontend) improve comments highlights #1961
|
- 💄(frontend) improve comments highlights #1961
|
||||||
|
- ♿️(frontend) fix empty heading before section titles in HTML export #2125
|
||||||
|
|
||||||
## [v4.8.3] - 2026-03-23
|
## [v4.8.3] - 2026-03-23
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- ✨(backend) improve indexing command with checkpoint recovery and asynchronicity
|
|
||||||
- ♿️(frontend) improve version history list accessibility #2033
|
- ♿️(frontend) improve version history list accessibility #2033
|
||||||
- ♿(frontend) focus skip link on headings and skip grid dropzone #1983
|
- ♿(frontend) focus skip link on headings and skip grid dropzone #1983
|
||||||
- ♿️(frontend) add sr-only format to export download button #2088
|
- ♿️(frontend) add sr-only format to export download button #2088
|
||||||
|
|||||||
@@ -1,52 +0,0 @@
|
|||||||
# Index Command
|
|
||||||
|
|
||||||
The `index` management command is used to index documents to the remote search indexer.
|
|
||||||
|
|
||||||
It sends an asynchronous task to the Celery worker.
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
### Command Line
|
|
||||||
|
|
||||||
```bash
|
|
||||||
python manage.py index \
|
|
||||||
--lower-time-bound "2024-01-01T00:00:00" \
|
|
||||||
--upper-time-bound "2024-01-31T23:59:59" \
|
|
||||||
--batch-size 200 \
|
|
||||||
--crash-safe-mode false
|
|
||||||
```
|
|
||||||
|
|
||||||
### Django Admin
|
|
||||||
|
|
||||||
The command is available in the Django admin interface:
|
|
||||||
|
|
||||||
1. Go to `/admin/`
|
|
||||||
2. Click on **"Run Indexing"** in the CORE section
|
|
||||||
3. Fill in the form with the desired parameters
|
|
||||||
4. Click **"Run Indexing Command"**
|
|
||||||
|
|
||||||
## Parameters
|
|
||||||
|
|
||||||
### `--batch-size`
|
|
||||||
- **type:** Integer
|
|
||||||
- **default:** `50`
|
|
||||||
- **description:** Number of documents to process per batch. Higher values may improve performance but use more memory.
|
|
||||||
|
|
||||||
### `--lower-time-bound`
|
|
||||||
- **optional**: true
|
|
||||||
- **type:** ISO 8601 datetime string
|
|
||||||
- **default:** `None`
|
|
||||||
- **description:** Only documents updated after this date will be indexed.
|
|
||||||
|
|
||||||
### `--upper-time-bound` (optional)
|
|
||||||
- **optional**: true
|
|
||||||
- **type:** ISO 8601 datetime string
|
|
||||||
- **default:** `None`
|
|
||||||
- **description:** Only documents updated before this date will be indexed.
|
|
||||||
|
|
||||||
### `--crash-safe-mode`
|
|
||||||
- **type:** Boolean flag
|
|
||||||
- **default:** `True`
|
|
||||||
- **description:** When enabled, the command orders documents by `updated_at` and stores the last indexed document's timestamp in cache. This allows resuming indexing from the last successful batch in case of a crash. However, it is more computationally expensive due to sorting.b
|
|
||||||
|
|
||||||
|
|
||||||
@@ -60,10 +60,13 @@
|
|||||||
"groupName": "ignored js dependencies",
|
"groupName": "ignored js dependencies",
|
||||||
"matchManagers": ["npm"],
|
"matchManagers": ["npm"],
|
||||||
"matchPackageNames": [
|
"matchPackageNames": [
|
||||||
|
"@react-pdf/renderer",
|
||||||
"fetch-mock",
|
"fetch-mock",
|
||||||
"node",
|
"node",
|
||||||
"node-fetch",
|
"node-fetch",
|
||||||
"react-resizable-panels",
|
"react-resizable-panels",
|
||||||
|
"stylelint",
|
||||||
|
"stylelint-config-standard",
|
||||||
"workbox-webpack-plugin"
|
"workbox-webpack-plugin"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,8 @@
|
|||||||
"""Admin classes and registrations for core app."""
|
"""Admin classes and registrations for core app."""
|
||||||
|
|
||||||
from django.conf import settings
|
|
||||||
from django.contrib import admin, messages
|
from django.contrib import admin, messages
|
||||||
from django.contrib.auth import admin as auth_admin
|
from django.contrib.auth import admin as auth_admin
|
||||||
from django.core.management import call_command
|
from django.shortcuts import redirect
|
||||||
from django.shortcuts import redirect, render
|
|
||||||
from django.utils.translation import gettext_lazy as _
|
from django.utils.translation import gettext_lazy as _
|
||||||
|
|
||||||
from treebeard.admin import TreeAdmin
|
from treebeard.admin import TreeAdmin
|
||||||
@@ -229,36 +227,3 @@ class InvitationAdmin(admin.ModelAdmin):
|
|||||||
def save_model(self, request, obj, form, change):
|
def save_model(self, request, obj, form, change):
|
||||||
obj.issuer = request.user
|
obj.issuer = request.user
|
||||||
obj.save()
|
obj.save()
|
||||||
|
|
||||||
|
|
||||||
@admin.register(models.RunIndexing)
|
|
||||||
class RunIndexingAdmin(admin.ModelAdmin):
|
|
||||||
"""Admin for running indexing commands."""
|
|
||||||
|
|
||||||
def changelist_view(self, request, extra_context=None):
|
|
||||||
"""Override to avoid querying the database and handle form submission."""
|
|
||||||
if request.method == "POST":
|
|
||||||
try:
|
|
||||||
call_command(
|
|
||||||
"index",
|
|
||||||
batch_size=int(request.POST.get("batch_size")),
|
|
||||||
lower_time_bound=request.POST.get("lower_time_bound"),
|
|
||||||
upper_time_bound=request.POST.get("upper_time_bound"),
|
|
||||||
crash_safe_mode=request.POST.get("crash_safe_mode"),
|
|
||||||
)
|
|
||||||
messages.success(request, _("Indexing triggered!"))
|
|
||||||
except Exception as e:
|
|
||||||
messages.error(
|
|
||||||
request,
|
|
||||||
_("Error running indexing command: %(error)s") % {"error": str(e)},
|
|
||||||
)
|
|
||||||
|
|
||||||
return render(
|
|
||||||
request=request,
|
|
||||||
template_name="runindexing.html",
|
|
||||||
context={
|
|
||||||
**self.admin_site.each_context(request),
|
|
||||||
"title": "Run Indexing Command",
|
|
||||||
"default_batch_size": settings.SEARCH_INDEXER_BATCH_SIZE,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|||||||
@@ -3,12 +3,11 @@ Handle search setup that needs to be done at bootstrap time.
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime
|
import time
|
||||||
|
|
||||||
from django.core.management.base import BaseCommand, CommandError
|
from django.core.management.base import BaseCommand, CommandError
|
||||||
|
|
||||||
from core.services.search_indexers import get_document_indexer
|
from core.services.search_indexers import get_document_indexer
|
||||||
from core.tasks.indexing import batch_document_indexer_task
|
|
||||||
|
|
||||||
logger = logging.getLogger("docs.search.bootstrap_search")
|
logger = logging.getLogger("docs.search.bootstrap_search")
|
||||||
|
|
||||||
@@ -28,31 +27,6 @@ class Command(BaseCommand):
|
|||||||
default=50,
|
default=50,
|
||||||
help="Indexation query batch size",
|
help="Indexation query batch size",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
|
||||||
"--lower-time-bound",
|
|
||||||
action="store",
|
|
||||||
dest="lower_time_bound",
|
|
||||||
type=datetime.fromisoformat,
|
|
||||||
default=None,
|
|
||||||
help="DateTime in ISO format. Only documents updated after this date will be indexed",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--upper-time-bound",
|
|
||||||
action="store",
|
|
||||||
dest="upper_time_bound",
|
|
||||||
type=datetime.fromisoformat,
|
|
||||||
default=None,
|
|
||||||
help="DateTime in ISO format. Only documents updated before this date will be indexed",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--crash-safe-mode",
|
|
||||||
action="store_true",
|
|
||||||
dest="crash_safe_mode",
|
|
||||||
default=True,
|
|
||||||
help="If True, order documents by updated_at and store the last indexed document.updated_at in cache. "
|
|
||||||
"This allows resuming indexing from the last successful batch in case of a crash "
|
|
||||||
"but is more computationally expensive due to sorting.",
|
|
||||||
)
|
|
||||||
|
|
||||||
def handle(self, *args, **options):
|
def handle(self, *args, **options):
|
||||||
"""Launch and log search index generation."""
|
"""Launch and log search index generation."""
|
||||||
@@ -61,18 +35,18 @@ class Command(BaseCommand):
|
|||||||
if not indexer:
|
if not indexer:
|
||||||
raise CommandError("The indexer is not enabled or properly configured.")
|
raise CommandError("The indexer is not enabled or properly configured.")
|
||||||
|
|
||||||
try:
|
logger.info("Starting to regenerate Find index...")
|
||||||
batch_document_indexer_task.apply_async(
|
start = time.perf_counter()
|
||||||
kwargs={
|
batch_size = options["batch_size"]
|
||||||
"lower_time_bound": options["lower_time_bound"],
|
|
||||||
"upper_time_bound": options["upper_time_bound"],
|
|
||||||
"batch_size": options["batch_size"],
|
|
||||||
"crash_safe_mode": options["crash_safe_mode"],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
except Exception as err:
|
|
||||||
raise CommandError("Unable send indexing task to worker") from err
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
count = indexer.index(batch_size=batch_size)
|
||||||
|
except Exception as err:
|
||||||
|
raise CommandError("Unable to regenerate index") from err
|
||||||
|
|
||||||
|
duration = time.perf_counter() - start
|
||||||
logger.info(
|
logger.info(
|
||||||
"Document indexing task sent to worker",
|
"Search index regenerated from %d document(s) in %.2f seconds.",
|
||||||
|
count,
|
||||||
|
duration,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,23 +0,0 @@
|
|||||||
# Generated by Django 5.2.12 on 2026-03-26 16:34
|
|
||||||
|
|
||||||
from django.db import migrations, models
|
|
||||||
|
|
||||||
|
|
||||||
class Migration(migrations.Migration):
|
|
||||||
|
|
||||||
dependencies = [
|
|
||||||
('core', '0031_clean_onboarding_accesses'),
|
|
||||||
]
|
|
||||||
|
|
||||||
operations = [
|
|
||||||
migrations.CreateModel(
|
|
||||||
name='RunIndexing',
|
|
||||||
fields=[
|
|
||||||
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
|
|
||||||
],
|
|
||||||
options={
|
|
||||||
'verbose_name': 'Run Indexing',
|
|
||||||
'managed': False,
|
|
||||||
},
|
|
||||||
),
|
|
||||||
]
|
|
||||||
@@ -2029,13 +2029,3 @@ class Invitation(BaseModel):
|
|||||||
"partial_update": is_admin_or_owner,
|
"partial_update": is_admin_or_owner,
|
||||||
"retrieve": is_admin_or_owner,
|
"retrieve": is_admin_or_owner,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
class RunIndexing(models.Model):
|
|
||||||
"""Proxy model for indexing management in admin."""
|
|
||||||
|
|
||||||
class Meta:
|
|
||||||
"""Meta options."""
|
|
||||||
|
|
||||||
managed = False
|
|
||||||
verbose_name = _("Run Indexing")
|
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
"""Document search index management utilities and indexers"""
|
"""Document search index management utilities and indexers"""
|
||||||
|
|
||||||
import itertools
|
|
||||||
import logging
|
import logging
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
@@ -8,7 +7,6 @@ from functools import cache
|
|||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.contrib.auth.models import AnonymousUser
|
from django.contrib.auth.models import AnonymousUser
|
||||||
from django.core.cache import cache as django_cache
|
|
||||||
from django.core.exceptions import ImproperlyConfigured
|
from django.core.exceptions import ImproperlyConfigured
|
||||||
from django.utils.module_loading import import_string
|
from django.utils.module_loading import import_string
|
||||||
|
|
||||||
@@ -20,9 +18,6 @@ from core.enums import SearchType
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
BULK_INDEXER_CHECKPOINT = "bulk-indexer-checkpoint"
|
|
||||||
|
|
||||||
|
|
||||||
@cache
|
@cache
|
||||||
def get_document_indexer():
|
def get_document_indexer():
|
||||||
"""Returns an instance of indexer service if enabled and properly configured."""
|
"""Returns an instance of indexer service if enabled and properly configured."""
|
||||||
@@ -130,7 +125,7 @@ class BaseDocumentIndexer(ABC):
|
|||||||
if not self.search_url:
|
if not self.search_url:
|
||||||
raise ImproperlyConfigured("SEARCH_URL must be set in Django settings.")
|
raise ImproperlyConfigured("SEARCH_URL must be set in Django settings.")
|
||||||
|
|
||||||
def index(self, queryset=None, batch_size=None, crash_safe_mode=False):
|
def index(self, queryset=None, batch_size=None):
|
||||||
"""
|
"""
|
||||||
Fetch documents in batches, serialize them, and push to the search backend.
|
Fetch documents in batches, serialize them, and push to the search backend.
|
||||||
|
|
||||||
@@ -139,37 +134,35 @@ class BaseDocumentIndexer(ABC):
|
|||||||
Defaults to all documents without filter.
|
Defaults to all documents without filter.
|
||||||
batch_size (int, optional): Number of documents per batch.
|
batch_size (int, optional): Number of documents per batch.
|
||||||
Defaults to settings.SEARCH_INDEXER_BATCH_SIZE.
|
Defaults to settings.SEARCH_INDEXER_BATCH_SIZE.
|
||||||
crash_safe_mode (bool, optional): If True, order documents by updated_at
|
|
||||||
and store the last indexed document.updated_at in cache.
|
|
||||||
This allows resuming indexing from the last successful batch in case of a crash
|
|
||||||
but is more computationally expensive due to sorting.
|
|
||||||
"""
|
"""
|
||||||
|
last_id = 0
|
||||||
count = 0
|
count = 0
|
||||||
queryset = queryset or models.Document.objects.all()
|
queryset = queryset or models.Document.objects.all()
|
||||||
batch_size = batch_size or self.batch_size
|
batch_size = batch_size or self.batch_size
|
||||||
|
|
||||||
if crash_safe_mode:
|
while True:
|
||||||
queryset = queryset.order_by("updated_at")
|
documents_batch = list(
|
||||||
|
queryset.filter(
|
||||||
|
id__gt=last_id,
|
||||||
|
).order_by("id")[:batch_size]
|
||||||
|
)
|
||||||
|
|
||||||
|
if not documents_batch:
|
||||||
|
break
|
||||||
|
|
||||||
for documents_batch in itertools.batched(queryset.iterator(), batch_size):
|
|
||||||
doc_paths = [doc.path for doc in documents_batch]
|
doc_paths = [doc.path for doc in documents_batch]
|
||||||
|
last_id = documents_batch[-1].id
|
||||||
accesses_by_document_path = get_batch_accesses_by_users_and_teams(doc_paths)
|
accesses_by_document_path = get_batch_accesses_by_users_and_teams(doc_paths)
|
||||||
|
|
||||||
serialized_batch = [
|
serialized_batch = [
|
||||||
self.serialize_document(document, accesses_by_document_path)
|
self.serialize_document(document, accesses_by_document_path)
|
||||||
for document in documents_batch
|
for document in documents_batch
|
||||||
if document.content or document.title
|
if document.content or document.title
|
||||||
]
|
]
|
||||||
|
|
||||||
if not serialized_batch:
|
if serialized_batch:
|
||||||
continue
|
self.push(serialized_batch)
|
||||||
|
count += len(serialized_batch)
|
||||||
self.push(serialized_batch)
|
|
||||||
count += len(serialized_batch)
|
|
||||||
|
|
||||||
if crash_safe_mode:
|
|
||||||
django_cache.set(
|
|
||||||
BULK_INDEXER_CHECKPOINT, serialized_batch[-1]["updated_at"]
|
|
||||||
)
|
|
||||||
|
|
||||||
return count
|
return count
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from django.db.models import signals
|
|||||||
from django.dispatch import receiver
|
from django.dispatch import receiver
|
||||||
|
|
||||||
from core import models
|
from core import models
|
||||||
from core.tasks.indexing import trigger_batch_document_indexer
|
from core.tasks.search import trigger_batch_document_indexer
|
||||||
from core.utils import get_users_sharing_documents_with_cache_key
|
from core.utils import get_users_sharing_documents_with_cache_key
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,130 +0,0 @@
|
|||||||
"""Trigger document indexation using celery task."""
|
|
||||||
|
|
||||||
from logging import getLogger
|
|
||||||
|
|
||||||
from django.conf import settings
|
|
||||||
from django.core.cache import cache
|
|
||||||
|
|
||||||
from django_redis.cache import RedisCache
|
|
||||||
|
|
||||||
from core import models
|
|
||||||
from core.services.search_indexers import (
|
|
||||||
get_document_indexer,
|
|
||||||
)
|
|
||||||
from core.utils import build_indexable_documents_queryset
|
|
||||||
|
|
||||||
from impress.celery_app import app
|
|
||||||
|
|
||||||
logger = getLogger(__file__)
|
|
||||||
|
|
||||||
|
|
||||||
@app.task
|
|
||||||
def document_indexer_task(document_id):
|
|
||||||
"""
|
|
||||||
Celery Task: Indexes a single document by its ID.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
document_id: Primary key of the document to index.
|
|
||||||
"""
|
|
||||||
indexer = get_document_indexer()
|
|
||||||
|
|
||||||
if indexer:
|
|
||||||
logger.info("Start document %s indexation", document_id)
|
|
||||||
indexer.index(models.Document.objects.filter(pk=document_id))
|
|
||||||
|
|
||||||
|
|
||||||
def batch_indexer_throttle_acquire(timeout: int = 0, atomic: bool = True):
|
|
||||||
"""
|
|
||||||
Acquire a throttle lock to prevent multiple batch indexation tasks during countdown.
|
|
||||||
|
|
||||||
implements a debouncing pattern: only the first call during the timeout period
|
|
||||||
will succeed, subsequent calls are skipped until the timeout expires.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
timeout (int): Lock duration in seconds (countdown period).
|
|
||||||
atomic (bool): Use Redis locks for atomic operations if available.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
bool: True if lock acquired (first call), False if already held (subsequent calls).
|
|
||||||
"""
|
|
||||||
key = "document-batch-indexer-throttle"
|
|
||||||
|
|
||||||
# Redis is used as cache database (not in tests). Use the lock feature here
|
|
||||||
# to ensure atomicity of changes to the throttle flag.
|
|
||||||
if isinstance(cache, RedisCache) and atomic:
|
|
||||||
with cache.locks(key):
|
|
||||||
return batch_indexer_throttle_acquire(timeout, atomic=False)
|
|
||||||
|
|
||||||
# cache.add() is atomic test-and-set operation:
|
|
||||||
# - If key doesn't exist: creates it with timeout and returns True
|
|
||||||
# - If key already exists: does nothing and returns False
|
|
||||||
# The key expires after timeout seconds, releasing the lock.
|
|
||||||
# The value 1 is irrelevant, only the key presence/absence matters.
|
|
||||||
return cache.add(key, 1, timeout=timeout)
|
|
||||||
|
|
||||||
|
|
||||||
@app.task
|
|
||||||
def batch_document_indexer_task(lower_time_bound=None, upper_time_bound=None, **kwargs):
|
|
||||||
"""
|
|
||||||
Celery Task: Batch indexes all documents modified since timestamp.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
lower_time_bound (datetime|isoformat, optional):
|
|
||||||
indexes documents updated or deleted after this timestamp.
|
|
||||||
upper_time_bound (datetime|isoformat, optional):
|
|
||||||
indexes documents updated or deleted before this timestamp.
|
|
||||||
"""
|
|
||||||
indexer = get_document_indexer()
|
|
||||||
|
|
||||||
if not indexer:
|
|
||||||
logger.warning("Indexing task triggered but no indexer configured: skipping")
|
|
||||||
return
|
|
||||||
|
|
||||||
count = indexer.index(
|
|
||||||
queryset=build_indexable_documents_queryset(
|
|
||||||
lower_time_bound=lower_time_bound, upper_time_bound=upper_time_bound
|
|
||||||
),
|
|
||||||
**kwargs,
|
|
||||||
)
|
|
||||||
logger.info("Indexed %d documents", count)
|
|
||||||
|
|
||||||
|
|
||||||
def trigger_batch_document_indexer(document):
|
|
||||||
"""
|
|
||||||
Trigger document indexation with optional debounce mechanism.
|
|
||||||
|
|
||||||
behavior depends on SEARCH_INDEXER_COUNTDOWN setting:
|
|
||||||
- if countdown > 0 sec (async batch mode):
|
|
||||||
* schedules a batch indexation task after countdown in seconds
|
|
||||||
* uses throttle mechanism to ensure only ONE batch task runs per countdown period
|
|
||||||
* all documents modified since first trigger are indexed together
|
|
||||||
- if countdown == 0 sec (sync mode):
|
|
||||||
* executes indexation synchronously in the current thread
|
|
||||||
* no batching, no throttling, no Celery task queuing
|
|
||||||
|
|
||||||
Args:
|
|
||||||
document (Document): the document instance that triggered the indexation.
|
|
||||||
"""
|
|
||||||
countdown = int(settings.SEARCH_INDEXER_COUNTDOWN)
|
|
||||||
|
|
||||||
# DO NOT create a task if indexation is disabled
|
|
||||||
if not settings.SEARCH_INDEXER_CLASS:
|
|
||||||
return
|
|
||||||
|
|
||||||
if countdown > 0:
|
|
||||||
# use throttle to ensure only one task is scheduled per countdown period.
|
|
||||||
# if throttle acquired, schedule batch task; otherwise skip.
|
|
||||||
if batch_indexer_throttle_acquire(timeout=countdown):
|
|
||||||
logger.info(
|
|
||||||
"Add task for batch document indexation from updated_at=%s in %d seconds",
|
|
||||||
document.updated_at.isoformat(),
|
|
||||||
countdown,
|
|
||||||
)
|
|
||||||
|
|
||||||
batch_document_indexer_task.apply_async(
|
|
||||||
kwargs={"lower_time_bound": document.updated_at}, countdown=countdown
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
logger.info("Skip task for batch document %s indexation", document.pk)
|
|
||||||
else:
|
|
||||||
document_indexer_task.apply(args=[document.pk])
|
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
"""Trigger document indexation using celery task."""
|
||||||
|
|
||||||
|
from logging import getLogger
|
||||||
|
|
||||||
|
from django.conf import settings
|
||||||
|
from django.core.cache import cache
|
||||||
|
from django.db.models import Q
|
||||||
|
|
||||||
|
from django_redis.cache import RedisCache
|
||||||
|
|
||||||
|
from core import models
|
||||||
|
from core.services.search_indexers import (
|
||||||
|
get_document_indexer,
|
||||||
|
)
|
||||||
|
|
||||||
|
from impress.celery_app import app
|
||||||
|
|
||||||
|
logger = getLogger(__file__)
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def document_indexer_task(document_id):
|
||||||
|
"""Celery Task : Sends indexation query for a document."""
|
||||||
|
indexer = get_document_indexer()
|
||||||
|
|
||||||
|
if indexer:
|
||||||
|
logger.info("Start document %s indexation", document_id)
|
||||||
|
indexer.index(models.Document.objects.filter(pk=document_id))
|
||||||
|
|
||||||
|
|
||||||
|
def batch_indexer_throttle_acquire(timeout: int = 0, atomic: bool = True):
|
||||||
|
"""
|
||||||
|
Enable the task throttle flag for a delay.
|
||||||
|
Uses redis locks if available to ensure atomic changes
|
||||||
|
"""
|
||||||
|
key = "document-batch-indexer-throttle"
|
||||||
|
|
||||||
|
# Redis is used as cache database (not in tests). Use the lock feature here
|
||||||
|
# to ensure atomicity of changes to the throttle flag.
|
||||||
|
if isinstance(cache, RedisCache) and atomic:
|
||||||
|
with cache.locks(key):
|
||||||
|
return batch_indexer_throttle_acquire(timeout, atomic=False)
|
||||||
|
|
||||||
|
# Use add() here :
|
||||||
|
# - set the flag and returns true if not exist
|
||||||
|
# - do nothing and return false if exist
|
||||||
|
return cache.add(key, 1, timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
|
@app.task
|
||||||
|
def batch_document_indexer_task(timestamp):
|
||||||
|
"""Celery Task : Sends indexation query for a batch of documents."""
|
||||||
|
indexer = get_document_indexer()
|
||||||
|
|
||||||
|
if indexer:
|
||||||
|
queryset = models.Document.objects.filter(
|
||||||
|
Q(updated_at__gte=timestamp)
|
||||||
|
| Q(deleted_at__gte=timestamp)
|
||||||
|
| Q(ancestors_deleted_at__gte=timestamp)
|
||||||
|
)
|
||||||
|
|
||||||
|
count = indexer.index(queryset)
|
||||||
|
logger.info("Indexed %d documents", count)
|
||||||
|
|
||||||
|
|
||||||
|
def trigger_batch_document_indexer(document):
|
||||||
|
"""
|
||||||
|
Trigger indexation task with debounce a delay set by the SEARCH_INDEXER_COUNTDOWN setting.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
document (Document): The document instance.
|
||||||
|
"""
|
||||||
|
countdown = int(settings.SEARCH_INDEXER_COUNTDOWN)
|
||||||
|
|
||||||
|
# DO NOT create a task if indexation if disabled
|
||||||
|
if not settings.SEARCH_INDEXER_CLASS:
|
||||||
|
return
|
||||||
|
|
||||||
|
if countdown > 0:
|
||||||
|
# Each time this method is called during a countdown, we increment the
|
||||||
|
# counter and each task decrease it, so the index be run only once.
|
||||||
|
if batch_indexer_throttle_acquire(timeout=countdown):
|
||||||
|
logger.info(
|
||||||
|
"Add task for batch document indexation from updated_at=%s in %d seconds",
|
||||||
|
document.updated_at.isoformat(),
|
||||||
|
countdown,
|
||||||
|
)
|
||||||
|
|
||||||
|
batch_document_indexer_task.apply_async(
|
||||||
|
args=[document.updated_at], countdown=countdown
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.info("Skip task for batch document %s indexation", document.pk)
|
||||||
|
else:
|
||||||
|
document_indexer_task.apply(args=[document.pk])
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
{% extends "admin/base_site.html" %}
|
|
||||||
{% load i18n %}
|
|
||||||
|
|
||||||
{% block content %}
|
|
||||||
|
|
||||||
<form method="POST" >
|
|
||||||
{% csrf_token %}
|
|
||||||
|
|
||||||
<hr style="margin-bottom: 10px;">
|
|
||||||
|
|
||||||
<p>
|
|
||||||
This command triggers the indexing of all documents within the specified time bound.
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
See <b>docs/commands/index.md</b> for more details.
|
|
||||||
</p>
|
|
||||||
|
|
||||||
<hr style="margin-bottom: 20px;">
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<label for="batch_size" style="margin-right: 10px">Batch size</label>
|
|
||||||
<input id="batch_size" type="number" name="batch_size" value={{ default_batch_size }} style="width: 80px;" >
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<label for="lower_time_bound" style="margin-right: 10px">Lower time bound (optional)</label>
|
|
||||||
<input id="lower_time_bound" type="datetime-local" name="lower_time_bound" >
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<label for="upper_time_bound" style="margin-right: 10px">Upper time bound (optional)</label>
|
|
||||||
<input id="upper_time_bound" type="datetime-local" name="upper_time_bound" >
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div>
|
|
||||||
<label for="crash_safe_mode" style="margin-right: 10px">Crash save mode</label>
|
|
||||||
<input id="crash_safe_mode" type="checkbox" name="crash_safe_mode" value="true" checked>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<input type="submit" value="{% translate 'Run Indexing' %}" style="margin-top: 20px;">
|
|
||||||
</form>
|
|
||||||
|
|
||||||
{% endblock %}
|
|
||||||
@@ -2,24 +2,21 @@
|
|||||||
Unit test for `index` command.
|
Unit test for `index` command.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from datetime import datetime, timedelta, timezone
|
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
from unittest import mock
|
from unittest import mock
|
||||||
|
|
||||||
from django.core.cache import cache
|
|
||||||
from django.core.management import CommandError, call_command
|
from django.core.management import CommandError, call_command
|
||||||
from django.db import transaction
|
from django.db import transaction
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core import factories
|
from core import factories
|
||||||
from core.services.search_indexers import BULK_INDEXER_CHECKPOINT, FindDocumentIndexer
|
from core.services.search_indexers import FindDocumentIndexer
|
||||||
from core.tests.conftest import create_document_with_updated_at
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
@pytest.mark.usefixtures("indexer_settings")
|
@pytest.mark.usefixtures("indexer_settings")
|
||||||
def test_index_without_bound_success():
|
def test_index():
|
||||||
"""Test the command `index` that run the Find app indexer for all the available documents."""
|
"""Test the command `index` that run the Find app indexer for all the available documents."""
|
||||||
user = factories.UserFactory()
|
user = factories.UserFactory()
|
||||||
indexer = FindDocumentIndexer()
|
indexer = FindDocumentIndexer()
|
||||||
@@ -40,152 +37,20 @@ def test_index_without_bound_success():
|
|||||||
}
|
}
|
||||||
|
|
||||||
with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
|
with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
|
||||||
call_command("index", crash_safe_mode=False)
|
call_command("index")
|
||||||
|
|
||||||
push_call_args = [call.args[0] for call in mock_push.call_args_list]
|
push_call_args = [call.args[0] for call in mock_push.call_args_list]
|
||||||
|
|
||||||
# called once but with a batch of docs
|
# called once but with a batch of docs
|
||||||
mock_push.assert_called_once()
|
mock_push.assert_called_once()
|
||||||
|
|
||||||
assert sorted(push_call_args[0], key=itemgetter("id")) == sorted(
|
assert sorted(push_call_args[0], key=itemgetter("id")) == sorted(
|
||||||
[
|
[
|
||||||
indexer.serialize_document(doc, accesses),
|
indexer.serialize_document(doc, accesses),
|
||||||
indexer.serialize_document(no_title_doc, accesses),
|
indexer.serialize_document(no_title_doc, accesses),
|
||||||
],
|
],
|
||||||
key=itemgetter("id"),
|
key=itemgetter("id"),
|
||||||
)
|
|
||||||
|
|
||||||
# crash_safe_mode deactivated -> no checkpoint stored
|
|
||||||
assert cache.get(BULK_INDEXER_CHECKPOINT) is None
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
@pytest.mark.usefixtures("indexer_settings")
|
|
||||||
def test_index_with_both_bounds_success():
|
|
||||||
"""Test the command `index` for all documents within time bound."""
|
|
||||||
cache.clear()
|
|
||||||
lower_time_bound = datetime(2024, 2, 1, tzinfo=timezone.utc)
|
|
||||||
upper_time_bound = lower_time_bound + timedelta(days=30)
|
|
||||||
|
|
||||||
document_too_early = create_document_with_updated_at(
|
|
||||||
lower_time_bound - timedelta(days=10)
|
|
||||||
)
|
|
||||||
document_in_window_1 = create_document_with_updated_at(
|
|
||||||
lower_time_bound + timedelta(days=5)
|
|
||||||
)
|
|
||||||
document_in_window_2 = create_document_with_updated_at(
|
|
||||||
lower_time_bound + timedelta(days=15)
|
|
||||||
)
|
|
||||||
document_too_late = create_document_with_updated_at(
|
|
||||||
upper_time_bound + timedelta(days=10)
|
|
||||||
)
|
|
||||||
|
|
||||||
with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
|
|
||||||
call_command(
|
|
||||||
"index",
|
|
||||||
lower_time_bound=lower_time_bound.isoformat(),
|
|
||||||
upper_time_bound=upper_time_bound.isoformat(),
|
|
||||||
)
|
)
|
||||||
all_push_call_args = [
|
|
||||||
document["id"]
|
|
||||||
for call_arg_list in mock_push.call_args_list
|
|
||||||
for document in call_arg_list.args[0]
|
|
||||||
]
|
|
||||||
|
|
||||||
# Only documents in window should be indexed
|
|
||||||
assert str(document_too_early.id) not in all_push_call_args
|
|
||||||
assert str(document_in_window_1.id) in all_push_call_args
|
|
||||||
assert str(document_in_window_2.id) in all_push_call_args
|
|
||||||
assert str(document_too_late.id) not in all_push_call_args
|
|
||||||
|
|
||||||
# Checkpoint should be set to last indexed document's updated_at
|
|
||||||
assert (
|
|
||||||
cache.get(BULK_INDEXER_CHECKPOINT)
|
|
||||||
== document_in_window_2.updated_at.isoformat()
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
|
||||||
@pytest.mark.usefixtures("indexer_settings")
|
|
||||||
def test_index_with_crash_recovery():
|
|
||||||
"""Test resuming indexing from checkpoint after a crash."""
|
|
||||||
cache.clear()
|
|
||||||
lower_time_bound = datetime(2024, 2, 1, tzinfo=timezone.utc)
|
|
||||||
upper_time_bound = lower_time_bound + timedelta(days=60)
|
|
||||||
|
|
||||||
batch_size = 2
|
|
||||||
documents = [
|
|
||||||
# batch 0
|
|
||||||
create_document_with_updated_at(lower_time_bound + timedelta(days=5)),
|
|
||||||
create_document_with_updated_at(lower_time_bound + timedelta(days=10)),
|
|
||||||
# batch 1
|
|
||||||
create_document_with_updated_at(lower_time_bound + timedelta(days=20)),
|
|
||||||
create_document_with_updated_at(lower_time_bound + timedelta(days=25)),
|
|
||||||
# batch 2 - will crash here
|
|
||||||
create_document_with_updated_at(lower_time_bound + timedelta(days=30)),
|
|
||||||
create_document_with_updated_at(lower_time_bound + timedelta(days=35)),
|
|
||||||
# batch 3
|
|
||||||
create_document_with_updated_at(lower_time_bound + timedelta(days=40)),
|
|
||||||
create_document_with_updated_at(lower_time_bound + timedelta(days=45)),
|
|
||||||
]
|
|
||||||
|
|
||||||
# First run: simulate crash on batch 3
|
|
||||||
with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
|
|
||||||
|
|
||||||
def push_with_failure(data):
|
|
||||||
# Crash when we encounter document at index 4 (batch 2 with batch_size=2)
|
|
||||||
if str(documents[4].id) in [document["id"] for document in data]:
|
|
||||||
raise ConnectionError("Simulated indexing error")
|
|
||||||
|
|
||||||
mock_push.side_effect = push_with_failure
|
|
||||||
|
|
||||||
call_command(
|
|
||||||
"index",
|
|
||||||
batch_size=batch_size,
|
|
||||||
lower_time_bound=lower_time_bound.isoformat(),
|
|
||||||
upper_time_bound=upper_time_bound.isoformat(),
|
|
||||||
)
|
|
||||||
all_push_call_args = [
|
|
||||||
document["id"]
|
|
||||||
for call_arg_list in mock_push.call_args_list
|
|
||||||
for document in call_arg_list.args[0]
|
|
||||||
]
|
|
||||||
|
|
||||||
# First 2 batches should have been indexed and checkpoint saved
|
|
||||||
checkpoint = cache.get(BULK_INDEXER_CHECKPOINT)
|
|
||||||
assert checkpoint == documents[3].updated_at.isoformat()
|
|
||||||
# first 2 batches should be indexed successfully
|
|
||||||
for i in range(0, 4):
|
|
||||||
assert str(documents[i].id) in all_push_call_args
|
|
||||||
# next batch should have been attempted but failed
|
|
||||||
for i in range(4, 6):
|
|
||||||
assert str(documents[i].id) in all_push_call_args
|
|
||||||
# last batches indexing should not have been attempted
|
|
||||||
for i in range(6, 8):
|
|
||||||
assert str(documents[i].id) not in all_push_call_args
|
|
||||||
|
|
||||||
# Second run: resume from checkpoint
|
|
||||||
with mock.patch.object(FindDocumentIndexer, "push") as mock_push:
|
|
||||||
call_command(
|
|
||||||
"index",
|
|
||||||
batch_size=batch_size,
|
|
||||||
lower_time_bound=checkpoint,
|
|
||||||
upper_time_bound=upper_time_bound.isoformat(),
|
|
||||||
)
|
|
||||||
all_push_call_args = [
|
|
||||||
document["id"]
|
|
||||||
for call_arg_list in mock_push.call_args_list
|
|
||||||
for document in call_arg_list.args[0]
|
|
||||||
]
|
|
||||||
|
|
||||||
# first 2 batches should NOT be re-indexed
|
|
||||||
# except the last document of the last batch which is on the checkpoint boundary
|
|
||||||
for i in range(0, 3):
|
|
||||||
assert str(documents[i].id) not in all_push_call_args
|
|
||||||
# next batches should be indexed including the document at the checkpoint boundary
|
|
||||||
# which has already been indexed and is re-indexed
|
|
||||||
for i in range(4, 8):
|
|
||||||
assert str(documents[i].id) in all_push_call_args
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.django_db
|
@pytest.mark.django_db
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from django.core.cache import cache
|
|||||||
import pytest
|
import pytest
|
||||||
import responses
|
import responses
|
||||||
|
|
||||||
from core import factories, models
|
from core import factories
|
||||||
from core.tests.utils.urls import reload_urls
|
from core.tests.utils.urls import reload_urls
|
||||||
|
|
||||||
USER = "user"
|
USER = "user"
|
||||||
@@ -143,10 +143,3 @@ def user_token():
|
|||||||
A fixture to create a user token for testing.
|
A fixture to create a user token for testing.
|
||||||
"""
|
"""
|
||||||
return build_authorization_bearer("some_token")
|
return build_authorization_bearer("some_token")
|
||||||
|
|
||||||
|
|
||||||
def create_document_with_updated_at(updated_at, **kwargs):
|
|
||||||
"""Helper to create a document with a specific updated_at, bypassing auto_now=True."""
|
|
||||||
document = factories.DocumentFactory(**kwargs)
|
|
||||||
models.Document.objects.filter(pk=document.pk).update(updated_at=updated_at)
|
|
||||||
return models.Document.objects.get(pk=document.pk)
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@
|
|||||||
|
|
||||||
import base64
|
import base64
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime, timedelta, timezone
|
|
||||||
|
|
||||||
from django.core.cache import cache
|
from django.core.cache import cache
|
||||||
|
|
||||||
@@ -10,7 +9,6 @@ import pycrdt
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from core import factories, utils
|
from core import factories, utils
|
||||||
from core.tests.conftest import create_document_with_updated_at
|
|
||||||
|
|
||||||
pytestmark = pytest.mark.django_db
|
pytestmark = pytest.mark.django_db
|
||||||
|
|
||||||
@@ -242,127 +240,3 @@ def test_utils_get_value_by_pattern_no_match():
|
|||||||
result = utils.get_value_by_pattern(data, r"^title\.")
|
result = utils.get_value_by_pattern(data, r"^title\.")
|
||||||
|
|
||||||
assert result == []
|
assert result == []
|
||||||
|
|
||||||
|
|
||||||
def test_utils_build_indexable_documents_queryset_no_filters():
|
|
||||||
"""Test build_indexable_documents_queryset with no filters returns all documents."""
|
|
||||||
factories.DocumentFactory.create_batch(3)
|
|
||||||
|
|
||||||
queryset = utils.build_indexable_documents_queryset()
|
|
||||||
|
|
||||||
assert queryset.count() == 3
|
|
||||||
|
|
||||||
|
|
||||||
def test_utils_build_indexable_documents_queryset_oldest_updated_at():
|
|
||||||
"""Test filtering by oldest_updated_at includes documents updated after timestamp."""
|
|
||||||
lower_time_bound = datetime(2024, 2, 1, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
document_too_old = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound - timedelta(days=30)
|
|
||||||
)
|
|
||||||
document_at_boundary = create_document_with_updated_at(lower_time_bound)
|
|
||||||
document_recent = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound + timedelta(days=15)
|
|
||||||
)
|
|
||||||
|
|
||||||
queryset = utils.build_indexable_documents_queryset(
|
|
||||||
lower_time_bound=lower_time_bound
|
|
||||||
)
|
|
||||||
|
|
||||||
assert queryset.count() == 2
|
|
||||||
assert document_too_old not in queryset
|
|
||||||
assert document_at_boundary in queryset
|
|
||||||
assert document_recent in queryset
|
|
||||||
|
|
||||||
|
|
||||||
def test_utils_build_indexable_documents_queryset_newest_updated_at():
|
|
||||||
"""Test filtering by newest_updated_at includes documents updated before timestamp."""
|
|
||||||
upper_time_bound = datetime(2024, 2, 1, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
document_old = create_document_with_updated_at(
|
|
||||||
updated_at=upper_time_bound - timedelta(days=30)
|
|
||||||
)
|
|
||||||
document_at_boundary = create_document_with_updated_at(upper_time_bound)
|
|
||||||
document_too_recent = create_document_with_updated_at(
|
|
||||||
updated_at=upper_time_bound + timedelta(days=15)
|
|
||||||
)
|
|
||||||
|
|
||||||
queryset = utils.build_indexable_documents_queryset(
|
|
||||||
upper_time_bound=upper_time_bound
|
|
||||||
)
|
|
||||||
|
|
||||||
assert queryset.count() == 2
|
|
||||||
assert document_old in queryset
|
|
||||||
assert document_at_boundary in queryset
|
|
||||||
assert document_too_recent not in queryset
|
|
||||||
|
|
||||||
|
|
||||||
def test_utils_build_indexable_documents_queryset_both_bounds():
|
|
||||||
"""Test filtering with both oldest and newest bounds."""
|
|
||||||
lower_time_bound = datetime(2024, 2, 1, tzinfo=timezone.utc)
|
|
||||||
upper_time_bound = lower_time_bound + timedelta(days=30)
|
|
||||||
|
|
||||||
document_too_early = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound - timedelta(days=10)
|
|
||||||
)
|
|
||||||
document_in_window = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound + timedelta(days=5)
|
|
||||||
)
|
|
||||||
other_document_in_window = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound + timedelta(days=15)
|
|
||||||
)
|
|
||||||
document_too_late = create_document_with_updated_at(
|
|
||||||
updated_at=upper_time_bound + timedelta(days=10)
|
|
||||||
)
|
|
||||||
|
|
||||||
queryset = utils.build_indexable_documents_queryset(
|
|
||||||
lower_time_bound=lower_time_bound, upper_time_bound=upper_time_bound
|
|
||||||
)
|
|
||||||
|
|
||||||
assert queryset.count() == 2
|
|
||||||
assert document_too_early not in queryset
|
|
||||||
assert document_in_window in queryset
|
|
||||||
assert other_document_in_window in queryset
|
|
||||||
assert document_too_late not in queryset
|
|
||||||
|
|
||||||
|
|
||||||
def test_utils_build_indexable_documents_queryset_deleted_at():
|
|
||||||
"""Test filtering includes documents with deleted_at in range."""
|
|
||||||
lower_time_bound = datetime(2024, 2, 1, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
document_deleted_in_range = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound - timedelta(days=30),
|
|
||||||
deleted_at=lower_time_bound + timedelta(days=15),
|
|
||||||
)
|
|
||||||
document_updated_before_range = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound - timedelta(days=30)
|
|
||||||
)
|
|
||||||
|
|
||||||
queryset = utils.build_indexable_documents_queryset(
|
|
||||||
lower_time_bound=lower_time_bound
|
|
||||||
)
|
|
||||||
|
|
||||||
assert queryset.count() == 1
|
|
||||||
assert document_deleted_in_range in queryset
|
|
||||||
assert document_updated_before_range not in queryset
|
|
||||||
|
|
||||||
|
|
||||||
def test_utils_build_indexable_documents_queryset_ancestors_deleted_at():
|
|
||||||
"""Test filtering includes documents with ancestors_deleted_at in range."""
|
|
||||||
lower_time_bound = datetime(2024, 2, 1, tzinfo=timezone.utc)
|
|
||||||
|
|
||||||
document_ancestor_deleted_in_range = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound - timedelta(days=30),
|
|
||||||
ancestors_deleted_at=lower_time_bound + timedelta(days=15),
|
|
||||||
)
|
|
||||||
document_updated_before_range = create_document_with_updated_at(
|
|
||||||
updated_at=lower_time_bound - timedelta(days=30)
|
|
||||||
)
|
|
||||||
|
|
||||||
queryset = utils.build_indexable_documents_queryset(
|
|
||||||
lower_time_bound=lower_time_bound
|
|
||||||
)
|
|
||||||
|
|
||||||
assert queryset.count() == 1
|
|
||||||
assert document_ancestor_deleted_in_range in queryset
|
|
||||||
assert document_updated_before_range not in queryset
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from collections import defaultdict
|
|||||||
|
|
||||||
from django.core.cache import cache
|
from django.core.cache import cache
|
||||||
from django.db import models as db
|
from django.db import models as db
|
||||||
from django.db.models import Q, Subquery
|
from django.db.models import Subquery
|
||||||
|
|
||||||
import pycrdt
|
import pycrdt
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
@@ -168,47 +168,3 @@ def users_sharing_documents_with(user):
|
|||||||
elapsed,
|
elapsed,
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def build_indexable_documents_queryset(lower_time_bound=None, upper_time_bound=None):
|
|
||||||
"""
|
|
||||||
Build a queryset of documents to index filtered by timestamp range.
|
|
||||||
|
|
||||||
filters out documents that have been updated, or deleted outside the time range.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
lower_time_bound (datetime, optional):
|
|
||||||
keeps documents updated or deleted after this timestamp.
|
|
||||||
upper_time_bound (datetime, optional):
|
|
||||||
keeps documents updated or deleted before this timestamp.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
QuerySet: Filtered Document queryset ready for indexation.
|
|
||||||
"""
|
|
||||||
|
|
||||||
queryset = models.Document.objects.all()
|
|
||||||
|
|
||||||
conditions = Q()
|
|
||||||
if lower_time_bound and upper_time_bound:
|
|
||||||
conditions = (
|
|
||||||
Q(updated_at__gte=lower_time_bound, updated_at__lte=upper_time_bound)
|
|
||||||
| Q(deleted_at__gte=lower_time_bound, deleted_at__lte=upper_time_bound)
|
|
||||||
| Q(
|
|
||||||
ancestors_deleted_at__gte=lower_time_bound,
|
|
||||||
ancestors_deleted_at__lte=upper_time_bound,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
elif lower_time_bound:
|
|
||||||
conditions = (
|
|
||||||
Q(updated_at__gte=lower_time_bound)
|
|
||||||
| Q(deleted_at__gte=lower_time_bound)
|
|
||||||
| Q(ancestors_deleted_at__gte=lower_time_bound)
|
|
||||||
)
|
|
||||||
elif upper_time_bound:
|
|
||||||
conditions = (
|
|
||||||
Q(updated_at__lte=upper_time_bound)
|
|
||||||
| Q(deleted_at__lte=upper_time_bound)
|
|
||||||
| Q(ancestors_deleted_at__lte=upper_time_bound)
|
|
||||||
)
|
|
||||||
|
|
||||||
return queryset.filter(conditions)
|
|
||||||
|
|||||||
@@ -47,9 +47,9 @@ test.describe('Doc AI feature', () => {
|
|||||||
|
|
||||||
await page.locator('.bn-block-outer').last().fill('Anything');
|
await page.locator('.bn-block-outer').last().fill('Anything');
|
||||||
await page.getByText('Anything').selectText();
|
await page.getByText('Anything').selectText();
|
||||||
expect(
|
await expect(
|
||||||
await page.locator('button[data-test="convertMarkdown"]').count(),
|
page.locator('button[data-test="convertMarkdown"]'),
|
||||||
).toBe(1);
|
).toHaveCount(1);
|
||||||
await expect(
|
await expect(
|
||||||
page.getByRole('button', { name: config.selector, exact: true }),
|
page.getByRole('button', { name: config.selector, exact: true }),
|
||||||
).toBeHidden();
|
).toBeHidden();
|
||||||
|
|||||||
@@ -155,13 +155,11 @@ test.describe('Doc Editor', () => {
|
|||||||
expect(wsClose.isClosed()).toBeTruthy();
|
expect(wsClose.isClosed()).toBeTruthy();
|
||||||
|
|
||||||
// Check the ws is connected again
|
// Check the ws is connected again
|
||||||
webSocketPromise = page.waitForEvent('websocket', (webSocket) => {
|
webSocket = await page.waitForEvent('websocket', (webSocket) => {
|
||||||
return webSocket
|
return webSocket
|
||||||
.url()
|
.url()
|
||||||
.includes('ws://localhost:4444/collaboration/ws/?room=');
|
.includes('ws://localhost:4444/collaboration/ws/?room=');
|
||||||
});
|
});
|
||||||
|
|
||||||
webSocket = await webSocketPromise;
|
|
||||||
framesentPromise = webSocket.waitForEvent('framesent');
|
framesentPromise = webSocket.waitForEvent('framesent');
|
||||||
framesent = await framesentPromise;
|
framesent = await framesentPromise;
|
||||||
expect(framesent.payload).not.toBeNull();
|
expect(framesent.payload).not.toBeNull();
|
||||||
@@ -578,12 +576,10 @@ test.describe('Doc Editor', () => {
|
|||||||
|
|
||||||
await page.reload();
|
await page.reload();
|
||||||
|
|
||||||
responseCanEditPromise = page.waitForResponse(
|
responseCanEdit = await page.waitForResponse(
|
||||||
(response) =>
|
(response) =>
|
||||||
response.url().includes(`/can-edit/`) && response.status() === 200,
|
response.url().includes(`/can-edit/`) && response.status() === 200,
|
||||||
);
|
);
|
||||||
|
|
||||||
responseCanEdit = await responseCanEditPromise;
|
|
||||||
expect(responseCanEdit.ok()).toBeTruthy();
|
expect(responseCanEdit.ok()).toBeTruthy();
|
||||||
|
|
||||||
jsonCanEdit = (await responseCanEdit.json()) as { can_edit: boolean };
|
jsonCanEdit = (await responseCanEdit.json()) as { can_edit: boolean };
|
||||||
|
|||||||
@@ -236,7 +236,7 @@ test.describe('Doc Header', () => {
|
|||||||
hasText: randomDoc,
|
hasText: randomDoc,
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(await row.count()).toBe(0);
|
await expect(row).toHaveCount(0);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('it checks the options available if administrator', async ({ page }) => {
|
test('it checks the options available if administrator', async ({ page }) => {
|
||||||
@@ -273,7 +273,7 @@ test.describe('Doc Header', () => {
|
|||||||
await expect(getMenuItem(page, 'Delete document')).toBeDisabled();
|
await expect(getMenuItem(page, 'Delete document')).toBeDisabled();
|
||||||
|
|
||||||
// Click somewhere else to close the options
|
// Click somewhere else to close the options
|
||||||
await page.click('body', { position: { x: 0, y: 0 } });
|
await page.locator('body').click({ position: { x: 0, y: 0 } });
|
||||||
|
|
||||||
await page.getByRole('button', { name: 'Share' }).click();
|
await page.getByRole('button', { name: 'Share' }).click();
|
||||||
|
|
||||||
@@ -348,7 +348,7 @@ test.describe('Doc Header', () => {
|
|||||||
await expect(getMenuItem(page, 'Delete document')).toBeDisabled();
|
await expect(getMenuItem(page, 'Delete document')).toBeDisabled();
|
||||||
|
|
||||||
// Click somewhere else to close the options
|
// Click somewhere else to close the options
|
||||||
await page.click('body', { position: { x: 0, y: 0 } });
|
await page.locator('body').click({ position: { x: 0, y: 0 } });
|
||||||
|
|
||||||
await page.getByRole('button', { name: 'Share' }).click();
|
await page.getByRole('button', { name: 'Share' }).click();
|
||||||
|
|
||||||
@@ -418,7 +418,7 @@ test.describe('Doc Header', () => {
|
|||||||
await expect(getMenuItem(page, 'Delete document')).toBeDisabled();
|
await expect(getMenuItem(page, 'Delete document')).toBeDisabled();
|
||||||
|
|
||||||
// Click somewhere else to close the options
|
// Click somewhere else to close the options
|
||||||
await page.click('body', { position: { x: 0, y: 0 } });
|
await page.locator('body').click({ position: { x: 0, y: 0 } });
|
||||||
|
|
||||||
await page.getByRole('button', { name: 'Share' }).click();
|
await page.getByRole('button', { name: 'Share' }).click();
|
||||||
|
|
||||||
|
|||||||
@@ -177,5 +177,5 @@ const dragAndDropFiles = async (
|
|||||||
return dt;
|
return dt;
|
||||||
}, filesData);
|
}, filesData);
|
||||||
|
|
||||||
await page.dispatchEvent(selector, 'drop', { dataTransfer });
|
await page.locator(selector).dispatchEvent('drop', { dataTransfer });
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -43,15 +43,12 @@ test.describe('Doc Tree', () => {
|
|||||||
await expect(secondSubPageItem).toBeVisible();
|
await expect(secondSubPageItem).toBeVisible();
|
||||||
|
|
||||||
// Check the position of the sub pages
|
// Check the position of the sub pages
|
||||||
const allSubPageItems = await docTree
|
const allSubPageItems = docTree.getByTestId(/^doc-sub-page-item/);
|
||||||
.getByTestId(/^doc-sub-page-item/)
|
await expect(allSubPageItems).toHaveCount(2);
|
||||||
.all();
|
|
||||||
|
|
||||||
expect(allSubPageItems.length).toBe(2);
|
|
||||||
|
|
||||||
// Check that elements are in the correct order
|
// Check that elements are in the correct order
|
||||||
await expect(allSubPageItems[0].getByText('first move')).toBeVisible();
|
await expect(allSubPageItems.nth(0).getByText('first move')).toBeVisible();
|
||||||
await expect(allSubPageItems[1].getByText('second move')).toBeVisible();
|
await expect(allSubPageItems.nth(1).getByText('second move')).toBeVisible();
|
||||||
|
|
||||||
// Will move the first sub page to the second position
|
// Will move the first sub page to the second position
|
||||||
const firstSubPageBoundingBox = await firstSubPageItem.boundingBox();
|
const firstSubPageBoundingBox = await firstSubPageItem.boundingBox();
|
||||||
@@ -91,17 +88,15 @@ test.describe('Doc Tree', () => {
|
|||||||
await expect(secondSubPageItem).toBeVisible();
|
await expect(secondSubPageItem).toBeVisible();
|
||||||
|
|
||||||
// Check that elements are in the correct order
|
// Check that elements are in the correct order
|
||||||
const allSubPageItemsAfterReload = await docTree
|
const allSubPageItemsAfterReload =
|
||||||
.getByTestId(/^doc-sub-page-item/)
|
docTree.getByTestId(/^doc-sub-page-item/);
|
||||||
.all();
|
await expect(allSubPageItemsAfterReload).toHaveCount(2);
|
||||||
|
|
||||||
expect(allSubPageItemsAfterReload.length).toBe(2);
|
|
||||||
|
|
||||||
await expect(
|
await expect(
|
||||||
allSubPageItemsAfterReload[0].getByText('second move'),
|
allSubPageItemsAfterReload.nth(0).getByText('second move'),
|
||||||
).toBeVisible();
|
).toBeVisible();
|
||||||
await expect(
|
await expect(
|
||||||
allSubPageItemsAfterReload[1].getByText('first move'),
|
allSubPageItemsAfterReload.nth(1).getByText('first move'),
|
||||||
).toBeVisible();
|
).toBeVisible();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -80,9 +80,9 @@ test.describe('Doc Version', () => {
|
|||||||
await expect(panel).toBeVisible();
|
await expect(panel).toBeVisible();
|
||||||
await expect(page.getByText('History', { exact: true })).toBeVisible();
|
await expect(page.getByText('History', { exact: true })).toBeVisible();
|
||||||
await expect(page.getByRole('status')).toBeHidden();
|
await expect(page.getByRole('status')).toBeHidden();
|
||||||
const items = await panel.locator('.version-item').all();
|
const items = panel.locator('.version-item');
|
||||||
expect(items.length).toBe(2);
|
await expect(items).toHaveCount(2);
|
||||||
await items[1].click();
|
await items.nth(1).click();
|
||||||
|
|
||||||
await expect(modal.getByText('Hello World')).toBeVisible();
|
await expect(modal.getByText('Hello World')).toBeVisible();
|
||||||
await expect(modal.getByText('It will create a version')).toBeHidden();
|
await expect(modal.getByText('It will create a version')).toBeHidden();
|
||||||
@@ -90,7 +90,7 @@ test.describe('Doc Version', () => {
|
|||||||
modal.locator('div[data-content-type="callout"]').first(),
|
modal.locator('div[data-content-type="callout"]').first(),
|
||||||
).toBeHidden();
|
).toBeHidden();
|
||||||
|
|
||||||
await items[0].click();
|
await items.nth(0).click();
|
||||||
|
|
||||||
await expect(modal.getByText('Hello World')).toBeVisible();
|
await expect(modal.getByText('Hello World')).toBeVisible();
|
||||||
await expect(modal.getByText('It will create a version')).toBeVisible();
|
await expect(modal.getByText('It will create a version')).toBeVisible();
|
||||||
@@ -101,7 +101,7 @@ test.describe('Doc Version', () => {
|
|||||||
modal.getByText('It will create a second version'),
|
modal.getByText('It will create a second version'),
|
||||||
).toBeHidden();
|
).toBeHidden();
|
||||||
|
|
||||||
await items[1].click();
|
await items.nth(1).click();
|
||||||
|
|
||||||
await expect(modal.getByText('Hello World')).toBeVisible();
|
await expect(modal.getByText('Hello World')).toBeVisible();
|
||||||
await expect(modal.getByText('It will create a version')).toBeHidden();
|
await expect(modal.getByText('It will create a version')).toBeHidden();
|
||||||
|
|||||||
@@ -3,6 +3,11 @@ import path from 'path';
|
|||||||
|
|
||||||
import { Locator, Page, TestInfo, expect } from '@playwright/test';
|
import { Locator, Page, TestInfo, expect } from '@playwright/test';
|
||||||
|
|
||||||
|
import theme_customization from '../../../../../backend/impress/configuration/theme/default.json';
|
||||||
|
|
||||||
|
export type BrowserName = 'chromium' | 'firefox' | 'webkit';
|
||||||
|
export const BROWSERS: BrowserName[] = ['chromium', 'webkit', 'firefox'];
|
||||||
|
|
||||||
/** Returns a locator for a menu item (handles both menuitem and menuitemradio roles) */
|
/** Returns a locator for a menu item (handles both menuitem and menuitemradio roles) */
|
||||||
export const getMenuItem = (
|
export const getMenuItem = (
|
||||||
context: Page | Locator,
|
context: Page | Locator,
|
||||||
@@ -13,11 +18,6 @@ export const getMenuItem = (
|
|||||||
.getByRole('menuitem', { name, exact: options?.exact })
|
.getByRole('menuitem', { name, exact: options?.exact })
|
||||||
.or(context.getByRole('menuitemradio', { name, exact: options?.exact }));
|
.or(context.getByRole('menuitemradio', { name, exact: options?.exact }));
|
||||||
|
|
||||||
import theme_customization from '../../../../../backend/impress/configuration/theme/default.json';
|
|
||||||
|
|
||||||
export type BrowserName = 'chromium' | 'firefox' | 'webkit';
|
|
||||||
export const BROWSERS: BrowserName[] = ['chromium', 'webkit', 'firefox'];
|
|
||||||
|
|
||||||
export const CONFIG = {
|
export const CONFIG = {
|
||||||
AI_BOT: {
|
AI_BOT: {
|
||||||
name: 'Docs AI',
|
name: 'Docs AI',
|
||||||
|
|||||||
@@ -23,7 +23,7 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@ag-media/react-pdf-table": "2.0.3",
|
"@ag-media/react-pdf-table": "2.0.3",
|
||||||
"@ai-sdk/openai": "3.0.19",
|
"@ai-sdk/openai": "3.0.45",
|
||||||
"@blocknote/code-block": "0.47.1",
|
"@blocknote/code-block": "0.47.1",
|
||||||
"@blocknote/core": "0.47.1",
|
"@blocknote/core": "0.47.1",
|
||||||
"@blocknote/mantine": "0.47.1",
|
"@blocknote/mantine": "0.47.1",
|
||||||
@@ -38,20 +38,20 @@
|
|||||||
"@emoji-mart/data": "1.2.1",
|
"@emoji-mart/data": "1.2.1",
|
||||||
"@emoji-mart/react": "1.1.1",
|
"@emoji-mart/react": "1.1.1",
|
||||||
"@fontsource-variable/inter": "5.2.8",
|
"@fontsource-variable/inter": "5.2.8",
|
||||||
"@fontsource-variable/material-symbols-outlined": "5.2.35",
|
"@fontsource-variable/material-symbols-outlined": "5.2.38",
|
||||||
"@fontsource/material-icons": "5.2.7",
|
"@fontsource/material-icons": "5.2.7",
|
||||||
"@gouvfr-lasuite/cunningham-react": "4.2.0",
|
"@gouvfr-lasuite/cunningham-react": "4.2.0",
|
||||||
"@gouvfr-lasuite/integration": "1.0.3",
|
"@gouvfr-lasuite/integration": "1.0.3",
|
||||||
"@gouvfr-lasuite/ui-kit": "0.19.6",
|
"@gouvfr-lasuite/ui-kit": "0.19.10",
|
||||||
"@hocuspocus/provider": "3.4.4",
|
"@hocuspocus/provider": "3.4.4",
|
||||||
"@mantine/core": "8.3.14",
|
"@mantine/core": "8.3.17",
|
||||||
"@mantine/hooks": "8.3.14",
|
"@mantine/hooks": "8.3.17",
|
||||||
"@react-aria/live-announcer": "3.4.4",
|
"@react-aria/live-announcer": "3.4.4",
|
||||||
"@react-pdf/renderer": "4.3.1",
|
"@react-pdf/renderer": "4.3.1",
|
||||||
"@sentry/nextjs": "10.38.0",
|
"@sentry/nextjs": "10.43.0",
|
||||||
"@tanstack/react-query": "5.90.21",
|
"@tanstack/react-query": "5.90.21",
|
||||||
"@tiptap/extensions": "*",
|
"@tiptap/extensions": "*",
|
||||||
"ai": "6.0.49",
|
"ai": "6.0.128",
|
||||||
"canvg": "4.0.3",
|
"canvg": "4.0.3",
|
||||||
"clsx": "2.1.1",
|
"clsx": "2.1.1",
|
||||||
"cmdk": "1.1.1",
|
"cmdk": "1.1.1",
|
||||||
@@ -59,28 +59,28 @@
|
|||||||
"emoji-datasource-apple": "16.0.0",
|
"emoji-datasource-apple": "16.0.0",
|
||||||
"emoji-mart": "5.6.0",
|
"emoji-mart": "5.6.0",
|
||||||
"emoji-regex": "10.6.0",
|
"emoji-regex": "10.6.0",
|
||||||
"i18next": "25.8.12",
|
"i18next": "25.8.18",
|
||||||
"i18next-browser-languagedetector": "8.2.1",
|
"i18next-browser-languagedetector": "8.2.1",
|
||||||
"idb": "8.0.3",
|
"idb": "8.0.3",
|
||||||
"lodash": "4.17.23",
|
"lodash": "4.17.23",
|
||||||
"luxon": "3.7.2",
|
"luxon": "3.7.2",
|
||||||
"next": "16.1.7",
|
"next": "16.1.7",
|
||||||
"posthog-js": "1.347.2",
|
"posthog-js": "1.360.2",
|
||||||
"react": "*",
|
"react": "*",
|
||||||
"react-aria-components": "1.15.1",
|
"react-aria-components": "1.16.0",
|
||||||
"react-dom": "*",
|
"react-dom": "*",
|
||||||
"react-dropzone": "15.0.0",
|
"react-dropzone": "15.0.0",
|
||||||
"react-i18next": "16.5.4",
|
"react-i18next": "16.5.8",
|
||||||
"react-intersection-observer": "10.0.2",
|
"react-intersection-observer": "10.0.3",
|
||||||
"react-resizable-panels": "3.0.6",
|
"react-resizable-panels": "3.0.6",
|
||||||
"react-select": "5.10.2",
|
"react-select": "5.10.2",
|
||||||
"styled-components": "6.3.9",
|
"styled-components": "6.3.11",
|
||||||
"use-debounce": "10.1.0",
|
"use-debounce": "10.1.0",
|
||||||
"uuid": "13.0.0",
|
"uuid": "13.0.0",
|
||||||
"y-protocols": "1.0.7",
|
"y-protocols": "1.0.7",
|
||||||
"yjs": "*",
|
"yjs": "*",
|
||||||
"zod": "3.25.28",
|
"zod": "4.3.6",
|
||||||
"zustand": "5.0.11"
|
"zustand": "5.0.12"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@svgr/webpack": "8.1.0",
|
"@svgr/webpack": "8.1.0",
|
||||||
@@ -89,26 +89,25 @@
|
|||||||
"@testing-library/jest-dom": "6.9.1",
|
"@testing-library/jest-dom": "6.9.1",
|
||||||
"@testing-library/react": "16.3.2",
|
"@testing-library/react": "16.3.2",
|
||||||
"@testing-library/user-event": "14.6.1",
|
"@testing-library/user-event": "14.6.1",
|
||||||
"@types/lodash": "4.17.23",
|
"@types/lodash": "4.17.24",
|
||||||
"@types/luxon": "3.7.1",
|
"@types/luxon": "3.7.1",
|
||||||
"@types/node": "*",
|
"@types/node": "*",
|
||||||
"@types/react": "*",
|
"@types/react": "*",
|
||||||
"@types/react-dom": "*",
|
"@types/react-dom": "*",
|
||||||
"@vitejs/plugin-react": "5.1.4",
|
"@vitejs/plugin-react": "6.0.1",
|
||||||
"cross-env": "10.1.0",
|
"cross-env": "10.1.0",
|
||||||
"dotenv": "17.3.1",
|
"dotenv": "17.3.1",
|
||||||
"eslint-plugin-docs": "*",
|
"eslint-plugin-docs": "*",
|
||||||
"fetch-mock": "9.11.0",
|
"fetch-mock": "9.11.0",
|
||||||
"jsdom": "28.1.0",
|
"jsdom": "29.0.0",
|
||||||
"node-fetch": "2.7.0",
|
"node-fetch": "2.7.0",
|
||||||
"prettier": "3.8.1",
|
"prettier": "3.8.1",
|
||||||
"stylelint": "16.26.1",
|
"stylelint": "16.26.1",
|
||||||
"stylelint-config-standard": "39.0.1",
|
"stylelint-config-standard": "39.0.1",
|
||||||
"stylelint-prettier": "5.0.3",
|
"stylelint-prettier": "5.0.3",
|
||||||
"typescript": "*",
|
"typescript": "*",
|
||||||
"vite-tsconfig-paths": "6.1.1",
|
"vitest": "4.1.0",
|
||||||
"vitest": "4.0.18",
|
"webpack": "5.105.4",
|
||||||
"webpack": "5.105.2",
|
|
||||||
"workbox-webpack-plugin": "7.1.0"
|
"workbox-webpack-plugin": "7.1.0"
|
||||||
},
|
},
|
||||||
"packageManager": "yarn@1.22.22"
|
"packageManager": "yarn@1.22.22"
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ import { useCallback, useEffect, useState } from 'react';
|
|||||||
import * as Y from 'yjs';
|
import * as Y from 'yjs';
|
||||||
|
|
||||||
import { useUpdateDoc } from '@/docs/doc-management/';
|
import { useUpdateDoc } from '@/docs/doc-management/';
|
||||||
import { KEY_LIST_DOC_VERSIONS } from '@/docs/doc-versioning';
|
import { KEY_LIST_DOC_VERSIONS } from '@/docs/doc-versioning/api/useDocVersions';
|
||||||
import { toBase64 } from '@/utils/string';
|
import { toBase64 } from '@/utils/string';
|
||||||
import { isFirefox } from '@/utils/userAgent';
|
import { isFirefox } from '@/utils/userAgent';
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,9 @@
|
|||||||
import { afterAll, afterEach, describe, expect, it, vi } from 'vitest';
|
import { afterAll, afterEach, describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
|
vi.mock('@/docs/doc-export/components/ModalExport', () => ({
|
||||||
|
ModalExport: vi.fn(),
|
||||||
|
}));
|
||||||
|
|
||||||
const originalEnv = process.env.NEXT_PUBLIC_PUBLISH_AS_MIT;
|
const originalEnv = process.env.NEXT_PUBLIC_PUBLISH_AS_MIT;
|
||||||
|
|
||||||
describe('useModuleExport', () => {
|
describe('useModuleExport', () => {
|
||||||
@@ -16,12 +21,12 @@ describe('useModuleExport', () => {
|
|||||||
const Export = await import('@/features/docs/doc-export/');
|
const Export = await import('@/features/docs/doc-export/');
|
||||||
|
|
||||||
expect(Export.default).toBeUndefined();
|
expect(Export.default).toBeUndefined();
|
||||||
}, 15000);
|
});
|
||||||
|
|
||||||
it('should load modules when NEXT_PUBLIC_PUBLISH_AS_MIT is false', async () => {
|
it('should load modules when NEXT_PUBLIC_PUBLISH_AS_MIT is false', async () => {
|
||||||
process.env.NEXT_PUBLIC_PUBLISH_AS_MIT = 'false';
|
process.env.NEXT_PUBLIC_PUBLISH_AS_MIT = 'false';
|
||||||
const Export = await import('@/features/docs/doc-export/');
|
const Export = await import('@/features/docs/doc-export/');
|
||||||
|
|
||||||
expect(Export.default).toHaveProperty('ModalExport');
|
expect(Export.default).toHaveProperty('ModalExport');
|
||||||
}, 15000);
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
+267
@@ -0,0 +1,267 @@
|
|||||||
|
import { improveHtmlAccessibility } from '../utils_html';
|
||||||
|
|
||||||
|
const parse = (html: string): Document => {
|
||||||
|
const parser = new DOMParser();
|
||||||
|
return parser.parseFromString(html, 'text/html');
|
||||||
|
};
|
||||||
|
|
||||||
|
const bodyHtml = (doc: Document) =>
|
||||||
|
doc.body.innerHTML.replace(/\s+/g, ' ').trim();
|
||||||
|
|
||||||
|
describe('improveHtmlAccessibility', () => {
|
||||||
|
// Headings
|
||||||
|
|
||||||
|
describe('headings', () => {
|
||||||
|
it('converts heading blocks without inner h tag to semantic h1–h6', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div data-content-type="heading" data-level="3"><span>Title</span></div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const h3 = doc.querySelector('h3');
|
||||||
|
expect(h3).not.toBeNull();
|
||||||
|
expect(h3!.textContent.trim()).toBe('Title');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not nest a second heading when BlockNote already outputs h1–h6', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div data-content-type="heading" data-level="2">
|
||||||
|
<h2 class="bn-inline-content">Section</h2>
|
||||||
|
</div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const h2 = doc.querySelectorAll('h2');
|
||||||
|
expect(h2).toHaveLength(1);
|
||||||
|
expect(h2[0].textContent).toBe('Section');
|
||||||
|
expect(h2[0].className).toBe('bn-inline-content');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses data-level when inner heading tag has a different level', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div data-content-type="heading" data-level="3">
|
||||||
|
<h2 class="bn-inline-content">Mismatch</h2>
|
||||||
|
</div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
expect(doc.querySelector('h3')).not.toBeNull();
|
||||||
|
expect(doc.querySelector('h2')).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('inserts an h1 with the document title when none exists', () => {
|
||||||
|
const doc = parse(`<p>Content</p>`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'My Doc');
|
||||||
|
|
||||||
|
const h1 = doc.querySelector('h1');
|
||||||
|
expect(h1).not.toBeNull();
|
||||||
|
expect(h1!.id).toBe('doc-title');
|
||||||
|
expect(h1!.textContent).toBe('My Doc');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not insert h1 when the document already has one', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div data-content-type="heading" data-level="1"><span>Existing</span></div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Ignored');
|
||||||
|
|
||||||
|
expect(doc.querySelectorAll('h1')).toHaveLength(1);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Lists
|
||||||
|
|
||||||
|
describe('lists', () => {
|
||||||
|
it('converts bullet list items to ul > li', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div class="bn-block-group">
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="bulletListItem"><span>A</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="bulletListItem"><span>B</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const items = doc.querySelectorAll('ul > li');
|
||||||
|
expect(items).toHaveLength(2);
|
||||||
|
expect(items[0].textContent).toBe('A');
|
||||||
|
expect(items[1].textContent).toBe('B');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('converts numbered list items to ol > li', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div class="bn-block-group">
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="numberedListItem"><span>1st</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="numberedListItem"><span>2nd</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
expect(doc.querySelectorAll('ol > li')).toHaveLength(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('splits lists when a heading sits between them', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div class="bn-block-group">
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="bulletListItem"><span>A</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="bulletListItem"><span>B</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="heading" data-level="2"><span>Next</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="bulletListItem"><span>C</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const uls = doc.querySelectorAll('ul');
|
||||||
|
expect(uls).toHaveLength(2);
|
||||||
|
expect(uls[0].querySelectorAll('li')).toHaveLength(2);
|
||||||
|
expect(uls[1].querySelectorAll('li')).toHaveLength(1);
|
||||||
|
|
||||||
|
const html = bodyHtml(doc);
|
||||||
|
expect(html.indexOf('<ul')).toBeLessThan(html.indexOf('<h2'));
|
||||||
|
expect(html.indexOf('<h2')).toBeLessThan(
|
||||||
|
html.indexOf('<ul', html.indexOf('<ul') + 1),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps consecutive same-type items in a single list', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div class="bn-block-group">
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="bulletListItem"><span>A</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="bulletListItem"><span>B</span></div>
|
||||||
|
</div>
|
||||||
|
<div class="bn-block-outer">
|
||||||
|
<div data-content-type="bulletListItem"><span>C</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
expect(doc.querySelectorAll('ul')).toHaveLength(1);
|
||||||
|
expect(doc.querySelectorAll('ul > li')).toHaveLength(3);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Quotes
|
||||||
|
|
||||||
|
it('converts quote blocks to blockquote', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div data-content-type="quote"><span>Wise words</span></div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const bq = doc.querySelector('blockquote');
|
||||||
|
expect(bq).not.toBeNull();
|
||||||
|
expect(bq!.textContent).toBe('Wise words');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Callouts
|
||||||
|
|
||||||
|
it('converts callout blocks to aside with role="note"', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div data-content-type="callout"><span>Note</span></div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const aside = doc.querySelector('aside');
|
||||||
|
expect(aside).not.toBeNull();
|
||||||
|
expect(aside!.getAttribute('role')).toBe('note');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Checklists
|
||||||
|
|
||||||
|
it('wraps check list items in a ul with role="list" and adds aria-checked', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div>
|
||||||
|
<div data-content-type="checkListItem">
|
||||||
|
<input type="checkbox" />
|
||||||
|
<span>Todo</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const ul = doc.querySelector('ul.checklist');
|
||||||
|
expect(ul).not.toBeNull();
|
||||||
|
expect(ul!.getAttribute('role')).toBe('list');
|
||||||
|
expect(doc.querySelector('input')!.getAttribute('aria-checked')).toBe(
|
||||||
|
'false',
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
// Code blocks
|
||||||
|
|
||||||
|
it('converts code blocks to pre > code and preserves data attributes', () => {
|
||||||
|
const doc = parse(`
|
||||||
|
<div data-content-type="codeBlock" class="hl-theme" data-language="js"><span>const x = 1;</span></div>
|
||||||
|
`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const pre = doc.querySelector('pre');
|
||||||
|
expect(pre).not.toBeNull();
|
||||||
|
expect(pre!.className).toBe('hl-theme');
|
||||||
|
expect(pre!.getAttribute('data-language')).toBe('js');
|
||||||
|
expect(pre!.querySelector('code')!.textContent.trim()).toBe('const x = 1;');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Images
|
||||||
|
|
||||||
|
it('adds empty alt to images without one', () => {
|
||||||
|
const doc = parse(`<img src="photo.png" />`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
expect(doc.querySelector('img')!.getAttribute('alt')).toBe('');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not overwrite an existing alt', () => {
|
||||||
|
const doc = parse(`<img src="photo.png" alt="A photo" />`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
expect(doc.querySelector('img')!.getAttribute('alt')).toBe('A photo');
|
||||||
|
});
|
||||||
|
|
||||||
|
// Article wrapper
|
||||||
|
|
||||||
|
it('wraps body in article with role="document"', () => {
|
||||||
|
const doc = parse(`<p>Hello</p>`);
|
||||||
|
|
||||||
|
improveHtmlAccessibility(doc, 'Doc');
|
||||||
|
|
||||||
|
const article = doc.querySelector('article');
|
||||||
|
expect(article).not.toBeNull();
|
||||||
|
expect(article!.getAttribute('role')).toBe('document');
|
||||||
|
expect(article!.getAttribute('aria-labelledby')).toBe('doc-title');
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -145,8 +145,20 @@ export const improveHtmlAccessibility = (
|
|||||||
headingBlocks.forEach((block) => {
|
headingBlocks.forEach((block) => {
|
||||||
const rawLevel = Number(block.getAttribute('data-level')) || 1;
|
const rawLevel = Number(block.getAttribute('data-level')) || 1;
|
||||||
const level = Math.min(Math.max(rawLevel, 1), 6);
|
const level = Math.min(Math.max(rawLevel, 1), 6);
|
||||||
const heading = parsedDocument.createElement(`h${level}`);
|
const tag = `h${level}`;
|
||||||
moveChildNodes(block, heading);
|
|
||||||
|
const existingHeading = block.querySelector('h1, h2, h3, h4, h5, h6');
|
||||||
|
const heading = parsedDocument.createElement(tag);
|
||||||
|
|
||||||
|
if (existingHeading) {
|
||||||
|
if (existingHeading.className) {
|
||||||
|
heading.className = existingHeading.className;
|
||||||
|
}
|
||||||
|
moveChildNodes(existingHeading, heading);
|
||||||
|
} else {
|
||||||
|
moveChildNodes(block, heading);
|
||||||
|
}
|
||||||
|
|
||||||
block.replaceWith(heading);
|
block.replaceWith(heading);
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -178,10 +190,11 @@ export const improveHtmlAccessibility = (
|
|||||||
listItem: HTMLElement;
|
listItem: HTMLElement;
|
||||||
contentType: string;
|
contentType: string;
|
||||||
level: number;
|
level: number;
|
||||||
|
blockOuterIndex: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
const listItemsInfo: ListItemInfo[] = [];
|
const listItemsInfo: ListItemInfo[] = [];
|
||||||
allBlockOuters.forEach((blockOuter) => {
|
allBlockOuters.forEach((blockOuter, index) => {
|
||||||
const listItem = blockOuter.querySelector<HTMLElement>(listItemSelector);
|
const listItem = blockOuter.querySelector<HTMLElement>(listItemSelector);
|
||||||
if (listItem) {
|
if (listItem) {
|
||||||
const contentType = listItem.getAttribute('data-content-type');
|
const contentType = listItem.getAttribute('data-content-type');
|
||||||
@@ -192,6 +205,7 @@ export const improveHtmlAccessibility = (
|
|||||||
listItem,
|
listItem,
|
||||||
contentType,
|
contentType,
|
||||||
level,
|
level,
|
||||||
|
blockOuterIndex: index,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -206,13 +220,19 @@ export const improveHtmlAccessibility = (
|
|||||||
const isBullet = contentType === 'bulletListItem';
|
const isBullet = contentType === 'bulletListItem';
|
||||||
const listTag = isBullet ? 'ul' : 'ol';
|
const listTag = isBullet ? 'ul' : 'ol';
|
||||||
|
|
||||||
// Check if previous item continues the same list (same type and level)
|
// Check if previous item continues the same list (same type, level, and
|
||||||
|
// no non-list block between them in the DOM : e.g. a heading separates lists).
|
||||||
const previousInfo = idx > 0 ? listItemsInfo[idx - 1] : null;
|
const previousInfo = idx > 0 ? listItemsInfo[idx - 1] : null;
|
||||||
|
const isAdjacentBlock = previousInfo && info.blockOuterIndex === previousInfo.blockOuterIndex + 1;
|
||||||
const continuesPreviousList =
|
const continuesPreviousList =
|
||||||
previousInfo &&
|
isAdjacentBlock &&
|
||||||
previousInfo.contentType === contentType &&
|
previousInfo.contentType === contentType &&
|
||||||
previousInfo.level === level;
|
previousInfo.level === level;
|
||||||
|
|
||||||
|
if (previousInfo && !isAdjacentBlock) {
|
||||||
|
listStack.length = 0;
|
||||||
|
}
|
||||||
|
|
||||||
// Find or create the appropriate list
|
// Find or create the appropriate list
|
||||||
let targetList: HTMLElement | null = null;
|
let targetList: HTMLElement | null = null;
|
||||||
|
|
||||||
|
|||||||
+3
-11
@@ -1,6 +1,4 @@
|
|||||||
import { render, screen } from '@testing-library/react';
|
import { render, screen } from '@testing-library/react';
|
||||||
import userEvent from '@testing-library/user-event';
|
|
||||||
import React from 'react';
|
|
||||||
import { afterAll, beforeEach, describe, expect, vi } from 'vitest';
|
import { afterAll, beforeEach, describe, expect, vi } from 'vitest';
|
||||||
|
|
||||||
import { AppWrapper } from '@/tests/utils';
|
import { AppWrapper } from '@/tests/utils';
|
||||||
@@ -40,17 +38,11 @@ describe('DocToolBox - Licence', () => {
|
|||||||
render(<DocToolBox doc={doc as any} />, {
|
render(<DocToolBox doc={doc as any} />, {
|
||||||
wrapper: AppWrapper,
|
wrapper: AppWrapper,
|
||||||
});
|
});
|
||||||
const optionsButton = await screen.findByLabelText('Export the document');
|
|
||||||
await userEvent.click(optionsButton);
|
|
||||||
|
|
||||||
// Wait for the export modal to be visible, then assert on its content text.
|
|
||||||
await screen.findByTestId('modal-export-title');
|
|
||||||
expect(
|
expect(
|
||||||
screen.getByText(
|
await screen.findByLabelText('Export the document'),
|
||||||
'Export your document to print or download in .docx, .odt, .pdf or .html(zip) format.',
|
|
||||||
),
|
|
||||||
).toBeInTheDocument();
|
).toBeInTheDocument();
|
||||||
}, 10000);
|
}, 15000);
|
||||||
|
|
||||||
test('The export button is not rendered when MIT version is activated', async () => {
|
test('The export button is not rendered when MIT version is activated', async () => {
|
||||||
process.env.NEXT_PUBLIC_PUBLISH_AS_MIT = 'true';
|
process.env.NEXT_PUBLIC_PUBLISH_AS_MIT = 'true';
|
||||||
@@ -68,5 +60,5 @@ describe('DocToolBox - Licence', () => {
|
|||||||
expect(
|
expect(
|
||||||
screen.queryByLabelText('Export the document'),
|
screen.queryByLabelText('Export the document'),
|
||||||
).not.toBeInTheDocument();
|
).not.toBeInTheDocument();
|
||||||
});
|
}, 15000);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
import { Button, useModal } from '@gouvfr-lasuite/cunningham-react';
|
import { Button, useModal } from '@gouvfr-lasuite/cunningham-react';
|
||||||
import { useTreeContext } from '@gouvfr-lasuite/ui-kit';
|
import { useTreeContext } from '@gouvfr-lasuite/ui-kit';
|
||||||
import { useQueryClient } from '@tanstack/react-query';
|
|
||||||
import dynamic from 'next/dynamic';
|
import dynamic from 'next/dynamic';
|
||||||
import { useRouter } from 'next/router';
|
import { useRouter } from 'next/router';
|
||||||
import { useEffect, useState } from 'react';
|
import { useState } from 'react';
|
||||||
import { useTranslation } from 'react-i18next';
|
import { useTranslation } from 'react-i18next';
|
||||||
import { css } from 'styled-components';
|
import { css } from 'styled-components';
|
||||||
|
|
||||||
@@ -39,7 +38,6 @@ import {
|
|||||||
useDocUtils,
|
useDocUtils,
|
||||||
useDuplicateDoc,
|
useDuplicateDoc,
|
||||||
} from '@/docs/doc-management';
|
} from '@/docs/doc-management';
|
||||||
import { KEY_LIST_DOC_VERSIONS } from '@/docs/doc-versioning';
|
|
||||||
import { useFocusStore, useResponsiveStore } from '@/stores';
|
import { useFocusStore, useResponsiveStore } from '@/stores';
|
||||||
|
|
||||||
import { useCopyCurrentEditorToClipboard } from '../hooks/useCopyCurrentEditorToClipboard';
|
import { useCopyCurrentEditorToClipboard } from '../hooks/useCopyCurrentEditorToClipboard';
|
||||||
@@ -88,7 +86,6 @@ interface DocToolBoxProps {
|
|||||||
export const DocToolBox = ({ doc }: DocToolBoxProps) => {
|
export const DocToolBox = ({ doc }: DocToolBoxProps) => {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
const treeContext = useTreeContext<Doc>();
|
const treeContext = useTreeContext<Doc>();
|
||||||
const queryClient = useQueryClient();
|
|
||||||
const router = useRouter();
|
const router = useRouter();
|
||||||
const { isChild, isTopRoot } = useDocUtils(doc);
|
const { isChild, isTopRoot } = useDocUtils(doc);
|
||||||
|
|
||||||
@@ -114,16 +111,6 @@ export const DocToolBox = ({ doc }: DocToolBoxProps) => {
|
|||||||
listInvalidQueries: [KEY_LIST_DOC, KEY_DOC, KEY_LIST_FAVORITE_DOC],
|
listInvalidQueries: [KEY_LIST_DOC, KEY_DOC, KEY_LIST_FAVORITE_DOC],
|
||||||
});
|
});
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
if (selectHistoryModal.isOpen) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
void queryClient.resetQueries({
|
|
||||||
queryKey: [KEY_LIST_DOC_VERSIONS],
|
|
||||||
});
|
|
||||||
}, [selectHistoryModal.isOpen, queryClient]);
|
|
||||||
|
|
||||||
// Emoji Management
|
// Emoji Management
|
||||||
const { emoji } = getEmojiAndTitle(doc.title ?? '');
|
const { emoji } = getEmojiAndTitle(doc.title ?? '');
|
||||||
const { updateDocEmoji } = useDocTitleUpdate();
|
const { updateDocEmoji } = useDocTitleUpdate();
|
||||||
|
|||||||
+3
-3
@@ -4,7 +4,7 @@ import {
|
|||||||
} from '@gouvfr-lasuite/cunningham-react';
|
} from '@gouvfr-lasuite/cunningham-react';
|
||||||
import { useTranslation } from 'react-i18next';
|
import { useTranslation } from 'react-i18next';
|
||||||
|
|
||||||
import { useEditorStore } from '../../doc-editor';
|
import { useEditorStore } from '@/docs/doc-editor/stores/useEditorStore';
|
||||||
|
|
||||||
export const useCopyCurrentEditorToClipboard = () => {
|
export const useCopyCurrentEditorToClipboard = () => {
|
||||||
const { editor } = useEditorStore();
|
const { editor } = useEditorStore();
|
||||||
@@ -21,8 +21,8 @@ export const useCopyCurrentEditorToClipboard = () => {
|
|||||||
try {
|
try {
|
||||||
const editorContentFormatted =
|
const editorContentFormatted =
|
||||||
asFormat === 'html'
|
asFormat === 'html'
|
||||||
? await editor.blocksToHTMLLossy()
|
? editor.blocksToHTMLLossy()
|
||||||
: await editor.blocksToMarkdownLossy();
|
: editor.blocksToMarkdownLossy();
|
||||||
await navigator.clipboard.writeText(editorContentFormatted);
|
await navigator.clipboard.writeText(editorContentFormatted);
|
||||||
const successMessage =
|
const successMessage =
|
||||||
asFormat === 'markdown'
|
asFormat === 'markdown'
|
||||||
|
|||||||
@@ -44,7 +44,7 @@ export function useUpdateDoc(queryConfig?: UseUpdateDoc) {
|
|||||||
...queryConfig,
|
...queryConfig,
|
||||||
onSuccess: (data, variables, onMutateResult, context) => {
|
onSuccess: (data, variables, onMutateResult, context) => {
|
||||||
queryConfig?.listInvalidQueries?.forEach((queryKey) => {
|
queryConfig?.listInvalidQueries?.forEach((queryKey) => {
|
||||||
void queryClient.invalidateQueries({
|
void queryClient.resetQueries({
|
||||||
queryKey: [queryKey],
|
queryKey: [queryKey],
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
+5
@@ -14,6 +14,11 @@ vi.mock('@/stores', () => ({
|
|||||||
}),
|
}),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
vi.mock('@gouvfr-lasuite/ui-kit', async () => ({
|
||||||
|
...(await vi.importActual('@gouvfr-lasuite/ui-kit')),
|
||||||
|
useTreeContext: () => null,
|
||||||
|
}));
|
||||||
|
|
||||||
describe('useDocTitleUpdate', () => {
|
describe('useDocTitleUpdate', () => {
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
vi.clearAllMocks();
|
||||||
|
|||||||
+5
-2
@@ -4,9 +4,12 @@ import { useEffect, useState } from 'react';
|
|||||||
import * as Y from 'yjs';
|
import * as Y from 'yjs';
|
||||||
|
|
||||||
import { Box, Text, TextErrors } from '@/components';
|
import { Box, Text, TextErrors } from '@/components';
|
||||||
import { BlockNoteReader, DocEditorContainer } from '@/docs/doc-editor/';
|
import { BlockNoteReader } from '@/docs/doc-editor/components/BlockNoteEditor';
|
||||||
|
import { DocEditorContainer } from '@/docs/doc-editor/components/DocEditor';
|
||||||
import { Doc, base64ToBlocknoteXmlFragment } from '@/docs/doc-management';
|
import { Doc, base64ToBlocknoteXmlFragment } from '@/docs/doc-management';
|
||||||
import { Versions, useDocVersion } from '@/docs/doc-versioning/';
|
|
||||||
|
import { useDocVersion } from '../api/useDocVersion';
|
||||||
|
import { Versions } from '../types';
|
||||||
|
|
||||||
import { DocVersionHeader } from './DocVersionHeader';
|
import { DocVersionHeader } from './DocVersionHeader';
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Doc } from '../doc-management';
|
import { Doc } from '../doc-management/types';
|
||||||
|
|
||||||
export interface APIListVersions {
|
export interface APIListVersions {
|
||||||
count: number;
|
count: number;
|
||||||
|
|||||||
+7
-3
@@ -1,4 +1,4 @@
|
|||||||
import { render, screen, waitFor } from '@testing-library/react';
|
import { act, render, screen, waitFor } from '@testing-library/react';
|
||||||
import fetchMock from 'fetch-mock';
|
import fetchMock from 'fetch-mock';
|
||||||
import i18next from 'i18next';
|
import i18next from 'i18next';
|
||||||
import { DateTime } from 'luxon';
|
import { DateTime } from 'luxon';
|
||||||
@@ -73,7 +73,9 @@ describe('DocsGridItemDate', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it(`should render rendered the updated_at field in the correct language`, async () => {
|
it(`should render rendered the updated_at field in the correct language`, async () => {
|
||||||
await i18next.changeLanguage('fr');
|
await act(async () => {
|
||||||
|
await i18next.changeLanguage('fr');
|
||||||
|
});
|
||||||
|
|
||||||
render(
|
render(
|
||||||
<DocsGridItemDate
|
<DocsGridItemDate
|
||||||
@@ -90,7 +92,9 @@ describe('DocsGridItemDate', () => {
|
|||||||
|
|
||||||
expect(screen.getByText('il y a 5 jours')).toBeInTheDocument();
|
expect(screen.getByText('il y a 5 jours')).toBeInTheDocument();
|
||||||
|
|
||||||
await i18next.changeLanguage('en');
|
await act(async () => {
|
||||||
|
await i18next.changeLanguage('en');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -1,12 +1,7 @@
|
|||||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
import { describe, expect, it, vi } from 'vitest';
|
||||||
|
|
||||||
vi.mock('@/../package.json', () => ({
|
|
||||||
default: { version: '0.0.0' },
|
|
||||||
}));
|
|
||||||
|
|
||||||
describe('DocsDB', () => {
|
describe('DocsDB', () => {
|
||||||
afterEach(() => {
|
beforeEach(() => {
|
||||||
vi.clearAllMocks();
|
|
||||||
vi.resetModules();
|
vi.resetModules();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -20,17 +15,16 @@ describe('DocsDB', () => {
|
|||||||
{ version: '3.0.0', expected: 3000000 },
|
{ version: '3.0.0', expected: 3000000 },
|
||||||
{ version: '10.20.30', expected: 10020030 },
|
{ version: '10.20.30', expected: 10020030 },
|
||||||
].forEach(({ version, expected }) => {
|
].forEach(({ version, expected }) => {
|
||||||
it(`correctly computes version for ${version}`, () => {
|
it(`correctly computes version for ${version}`, async () => {
|
||||||
vi.doMock('@/../package.json', () => ({
|
vi.doMock('@/../package.json', () => ({
|
||||||
default: { version },
|
default: { version },
|
||||||
}));
|
}));
|
||||||
|
|
||||||
return vi.importActual('../DocsDB').then((module: any) => {
|
const module = await import('../DocsDB');
|
||||||
const result = module.getCurrentVersion();
|
const result = (module as any).getCurrentVersion();
|
||||||
expect(result).toBe(expected);
|
expect(result).toBe(expected);
|
||||||
expect(result).toBeGreaterThan(previousExpected);
|
expect(result).toBeGreaterThan(previousExpected);
|
||||||
previousExpected = result;
|
previousExpected = result;
|
||||||
});
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,16 +1,9 @@
|
|||||||
/// <reference types="vitest" />
|
/// <reference types="vitest" />
|
||||||
import react from '@vitejs/plugin-react';
|
import react from '@vitejs/plugin-react';
|
||||||
import tsconfigPaths from 'vite-tsconfig-paths';
|
|
||||||
import { defineConfig } from 'vitest/config';
|
import { defineConfig } from 'vitest/config';
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
plugins: [
|
plugins: [react()],
|
||||||
react(),
|
|
||||||
tsconfigPaths({
|
|
||||||
root: '.',
|
|
||||||
projects: ['./tsconfig.json'],
|
|
||||||
}),
|
|
||||||
],
|
|
||||||
test: {
|
test: {
|
||||||
globals: true,
|
globals: true,
|
||||||
environment: 'jsdom',
|
environment: 'jsdom',
|
||||||
@@ -22,4 +15,7 @@ export default defineConfig({
|
|||||||
define: {
|
define: {
|
||||||
'process.env.NODE_ENV': 'test',
|
'process.env.NODE_ENV': 'test',
|
||||||
},
|
},
|
||||||
|
resolve: {
|
||||||
|
tsconfigPaths: true,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -32,17 +32,17 @@
|
|||||||
},
|
},
|
||||||
"resolutions": {
|
"resolutions": {
|
||||||
"@tiptap/extensions": "3.19.0",
|
"@tiptap/extensions": "3.19.0",
|
||||||
"@types/node": "24.10.13",
|
"@types/node": "24.12.0",
|
||||||
"@types/react": "19.2.14",
|
"@types/react": "19.2.14",
|
||||||
"@types/react-dom": "19.2.3",
|
"@types/react-dom": "19.2.3",
|
||||||
"eslint": "10.0.1",
|
"eslint": "10.0.3",
|
||||||
"glob": "13.0.6",
|
"glob": "13.0.6",
|
||||||
"prosemirror-view": "1.41.6",
|
"prosemirror-view": "1.41.6",
|
||||||
"react": "19.2.4",
|
"react": "19.2.4",
|
||||||
"react-dom": "19.2.4",
|
"react-dom": "19.2.4",
|
||||||
"typescript": "5.9.3",
|
"typescript": "5.9.3",
|
||||||
"wrap-ansi": "9.0.2",
|
"wrap-ansi": "10.0.0",
|
||||||
"yjs": "13.6.29"
|
"yjs": "13.6.30"
|
||||||
},
|
},
|
||||||
"packageManager": "yarn@1.22.22"
|
"packageManager": "yarn@1.22.22"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ const js = require('@eslint/js');
|
|||||||
const nextPlugin = require('@next/eslint-plugin-next');
|
const nextPlugin = require('@next/eslint-plugin-next');
|
||||||
const tanstackQuery = require('@tanstack/eslint-plugin-query');
|
const tanstackQuery = require('@tanstack/eslint-plugin-query');
|
||||||
const { defineConfig } = require('eslint/config');
|
const { defineConfig } = require('eslint/config');
|
||||||
const importPlugin = require('eslint-plugin-import');
|
const importPlugin = require('eslint-plugin-import-x');
|
||||||
const jsxA11y = require('eslint-plugin-jsx-a11y');
|
const jsxA11y = require('eslint-plugin-jsx-a11y');
|
||||||
const prettier = require('eslint-plugin-prettier');
|
const prettier = require('eslint-plugin-prettier');
|
||||||
const react = require('eslint-plugin-react');
|
const react = require('eslint-plugin-react');
|
||||||
|
|||||||
@@ -18,22 +18,22 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@eslint/js": "10.0.1",
|
"@eslint/js": "10.0.1",
|
||||||
"@next/eslint-plugin-next": "16.1.6",
|
"@next/eslint-plugin-next": "16.1.7",
|
||||||
"@tanstack/eslint-plugin-query": "5.91.4",
|
"@tanstack/eslint-plugin-query": "5.91.4",
|
||||||
"@typescript-eslint/eslint-plugin": "8.56.0",
|
"@typescript-eslint/eslint-plugin": "8.57.1",
|
||||||
"@typescript-eslint/parser": "8.56.0",
|
"@typescript-eslint/parser": "8.57.1",
|
||||||
"@typescript-eslint/utils": "8.56.0",
|
"@typescript-eslint/utils": "8.57.1",
|
||||||
"@vitest/eslint-plugin": "1.6.9",
|
"@vitest/eslint-plugin": "1.6.12",
|
||||||
"eslint-config-next": "16.1.6",
|
"eslint-config-next": "16.1.7",
|
||||||
"eslint-config-prettier": "10.1.8",
|
"eslint-config-prettier": "10.1.8",
|
||||||
"eslint-plugin-import": "2.32.0",
|
"eslint-plugin-import-x": "4.16.2",
|
||||||
"eslint-plugin-jest": "29.15.0",
|
"eslint-plugin-jest": "29.15.0",
|
||||||
"eslint-plugin-jsx-a11y": "6.10.2",
|
"eslint-plugin-jsx-a11y": "6.10.2",
|
||||||
"eslint-plugin-playwright": "2.5.1",
|
"eslint-plugin-playwright": "2.10.0",
|
||||||
"eslint-plugin-prettier": "5.5.5",
|
"eslint-plugin-prettier": "5.5.5",
|
||||||
"eslint-plugin-react": "7.37.5",
|
"eslint-plugin-react": "7.37.5",
|
||||||
"eslint-plugin-react-hooks": "5.2.0",
|
"eslint-plugin-react-hooks": "7.0.1",
|
||||||
"eslint-plugin-testing-library": "7.15.4",
|
"eslint-plugin-testing-library": "7.16.0",
|
||||||
"prettier": "3.8.1"
|
"prettier": "3.8.1"
|
||||||
},
|
},
|
||||||
"packageManager": "yarn@1.22.22"
|
"packageManager": "yarn@1.22.22"
|
||||||
|
|||||||
@@ -19,8 +19,8 @@
|
|||||||
"@types/node": "*",
|
"@types/node": "*",
|
||||||
"eslint-plugin-docs": "*",
|
"eslint-plugin-docs": "*",
|
||||||
"eslint-plugin-import": "2.32.0",
|
"eslint-plugin-import": "2.32.0",
|
||||||
"i18next-parser": "9.3.0",
|
"i18next-parser": "9.4.0",
|
||||||
"jest": "30.2.0",
|
"jest": "30.3.0",
|
||||||
"ts-jest": "29.4.6",
|
"ts-jest": "29.4.6",
|
||||||
"typescript": "*",
|
"typescript": "*",
|
||||||
"yargs": "18.0.0"
|
"yargs": "18.0.0"
|
||||||
|
|||||||
@@ -18,10 +18,10 @@
|
|||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@blocknote/server-util": "0.47.1",
|
"@blocknote/server-util": "0.47.1",
|
||||||
"@hocuspocus/server": "3.4.4",
|
"@hocuspocus/server": "3.4.4",
|
||||||
"@sentry/node": "10.38.0",
|
"@sentry/node": "10.43.0",
|
||||||
"@sentry/profiling-node": "10.38.0",
|
"@sentry/profiling-node": "10.43.0",
|
||||||
"@tiptap/extensions": "*",
|
"@tiptap/extensions": "*",
|
||||||
"axios": "1.13.5",
|
"axios": "1.13.6",
|
||||||
"cors": "2.8.6",
|
"cors": "2.8.6",
|
||||||
"express": "5.2.1",
|
"express": "5.2.1",
|
||||||
"express-ws": "5.0.2",
|
"express-ws": "5.0.2",
|
||||||
@@ -36,16 +36,16 @@
|
|||||||
"@types/express": "5.0.6",
|
"@types/express": "5.0.6",
|
||||||
"@types/express-ws": "3.0.6",
|
"@types/express-ws": "3.0.6",
|
||||||
"@types/node": "*",
|
"@types/node": "*",
|
||||||
"@types/supertest": "6.0.3",
|
"@types/supertest": "7.2.0",
|
||||||
"@types/ws": "8.18.1",
|
"@types/ws": "8.18.1",
|
||||||
"cross-env": "10.1.0",
|
"cross-env": "10.1.0",
|
||||||
"eslint-plugin-docs": "*",
|
"eslint-plugin-docs": "*",
|
||||||
"nodemon": "3.1.11",
|
"nodemon": "3.1.14",
|
||||||
"supertest": "7.2.2",
|
"supertest": "7.2.2",
|
||||||
"ts-node": "10.9.2",
|
"ts-node": "10.9.2",
|
||||||
"tsc-alias": "1.8.16",
|
"tsc-alias": "1.8.16",
|
||||||
"typescript": "*",
|
"typescript": "*",
|
||||||
"vitest": "4.0.18",
|
"vitest": "4.1.0",
|
||||||
"vitest-mock-extended": "3.1.0",
|
"vitest-mock-extended": "3.1.0",
|
||||||
"ws": "8.19.0"
|
"ws": "8.19.0"
|
||||||
},
|
},
|
||||||
|
|||||||
+2350
-2037
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,9 @@
|
|||||||
"@html-to/text-cli": "0.5.4",
|
"@html-to/text-cli": "0.5.4",
|
||||||
"mjml": "4.18.0"
|
"mjml": "4.18.0"
|
||||||
},
|
},
|
||||||
|
"resolutions": {
|
||||||
|
"minimatch": "^9.0.7"
|
||||||
|
},
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build-mjml-to-html": "bash ./bin/mjml-to-html",
|
"build-mjml-to-html": "bash ./bin/mjml-to-html",
|
||||||
|
|||||||
+6
-13
@@ -110,7 +110,7 @@ boolbase@^1.0.0:
|
|||||||
resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
|
resolved "https://registry.yarnpkg.com/boolbase/-/boolbase-1.0.0.tgz#68dff5fbe60c51eb37725ea9e3ed310dcc1e776e"
|
||||||
integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==
|
integrity sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==
|
||||||
|
|
||||||
brace-expansion@^2.0.1:
|
brace-expansion@^2.0.2:
|
||||||
version "2.0.2"
|
version "2.0.2"
|
||||||
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.2.tgz#54fc53237a613d854c7bd37463aad17df87214e7"
|
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.2.tgz#54fc53237a613d854c7bd37463aad17df87214e7"
|
||||||
integrity sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==
|
integrity sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==
|
||||||
@@ -562,19 +562,12 @@ mime@^2.4.6:
|
|||||||
resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367"
|
resolved "https://registry.yarnpkg.com/mime/-/mime-2.6.0.tgz#a2a682a95cd4d0cb1d6257e28f83da7e35800367"
|
||||||
integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==
|
integrity sha512-USPkMeET31rOMiarsBNIHZKLGgvKc/LrjofAnBlOttf5ajRvqiRA8QsenbcooctK6d6Ts6aqZXBA+XbkKthiQg==
|
||||||
|
|
||||||
minimatch@9.0.1:
|
minimatch@9.0.1, minimatch@^9.0.3, minimatch@^9.0.4, minimatch@^9.0.7:
|
||||||
version "9.0.1"
|
version "9.0.9"
|
||||||
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.1.tgz#8a555f541cf976c622daf078bb28f29fb927c253"
|
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.9.tgz#9b0cb9fcb78087f6fd7eababe2511c4d3d60574e"
|
||||||
integrity sha512-0jWhJpD/MdhPXwPuiRkCbfYfSKp2qnn2eOc279qI7f+osl/l+prKSrvhg157zSYvx/1nmgn2NqdT6k2Z7zSH9w==
|
integrity sha512-OBwBN9AL4dqmETlpS2zasx+vTeWclWzkblfZk7KTA5j3jeOONz/tRCnZomUyvNg83wL5Zv9Ss6HMJXAgL8R2Yg==
|
||||||
dependencies:
|
dependencies:
|
||||||
brace-expansion "^2.0.1"
|
brace-expansion "^2.0.2"
|
||||||
|
|
||||||
minimatch@^9.0.3, minimatch@^9.0.4:
|
|
||||||
version "9.0.5"
|
|
||||||
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5"
|
|
||||||
integrity sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==
|
|
||||||
dependencies:
|
|
||||||
brace-expansion "^2.0.1"
|
|
||||||
|
|
||||||
"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2:
|
"minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2:
|
||||||
version "7.1.2"
|
version "7.1.2"
|
||||||
|
|||||||
Reference in New Issue
Block a user