⚗️(backend) function to extract text from base64 yjs document

Function to extract text from base64 yjs document.
Can be usefull if we need to index the content
of the documents.

CHANGELOG.md

@@ -11,20 +11,27 @@ and this project adheres to
 
 ## Added
 
+- ⚗️(backend) Extract text from base64 yjs document #270
+
+
+## [1.4.0] - 2024-09-17
+
+## Added
+
 - ✨Add link public/authenticated/restricted access with read/editor roles #234
 - ✨(frontend) add copy link button #235
 - 🛂(frontend) access public docs without being logged #235
 
 ## Changed
 
-- ♻️ Allow null titles on documents for easier creation #234
+- ♻️(backend) Allow null titles on documents for easier creation #234
 - 🛂(backend) stop to list public doc to everyone #234
 - 🚚(frontend) change visibility in share modal #235
 - ⚡️(frontend) Improve summary #244
 
 ## Fixed
 
-- 🐛 Fix forcing ID when creating a document via API endpoint #234
+- 🐛(backend) Fix forcing ID when creating a document via API endpoint #234
 - 🐛 Rebuild frontend dev container from makefile #248
 
 
@@ -150,7 +157,8 @@ and this project adheres to
 - 🚀 Impress, project to manage your documents easily and collaboratively.
 
 
-[unreleased]: https://github.com/numerique-gouv/impress/compare/v1.3.0...main
+[unreleased]: https://github.com/numerique-gouv/impress/compare/v1.4.0...main
+[1.4.0]: https://github.com/numerique-gouv/impress/releases/v1.4.0
 [1.3.0]: https://github.com/numerique-gouv/impress/releases/v1.3.0
 [1.2.1]: https://github.com/numerique-gouv/impress/releases/v1.2.1
 [1.2.0]: https://github.com/numerique-gouv/impress/releases/v1.2.0

src/backend/core/tests/test_utils.py

@@ -10,7 +10,7 @@ from django.core import mail
 
 import pytest
 
-from core.utils import email_invitation
+from core.utils import email_invitation, yjs_base64_to_text
 
 pytestmark = pytest.mark.django_db
 
@@ -85,3 +85,29 @@ def test_utils__email_invitation_failed(mock_logger, _mock_send_mail):
 
     assert email == "guest@example.com"
     assert isinstance(exception, smtplib.SMTPException)
+
+
+def test_yjs_base64_to_text():
+    """
+    Test extract_text_from_saved_yjs_document
+    This base64 string is an example of what is saved in the database.
+    This base64 is generated from the blocknote editor, it contains
+    the text \n# *Hello* \n- w**or**ld
+    """
+    base64_string = (
+        "AR717vLVDgAHAQ5kb2N1bWVudC1zdG9yZQMKYmxvY2tHcm91cAcA9e7y1Q4AAw5ibG9ja0NvbnRh"
+        "aW5lcgcA9e7y1Q4BAwdoZWFkaW5nBwD17vLVDgIGBgD17vLVDgMGaXRhbGljAnt9hPXu8tUOBAVI"
+        "ZWxsb4b17vLVDgkGaXRhbGljBG51bGwoAPXu8tUOAg10ZXh0QWxpZ25tZW50AXcEbGVmdCgA9e7y"
+        "1Q4CBWxldmVsAX0BKAD17vLVDgECaWQBdyQwNGQ2MjM0MS04MzI2LTQyMzYtYTA4My00ODdlMjZm"
+        "YWQyMzAoAPXu8tUOAQl0ZXh0Q29sb3IBdwdkZWZhdWx0KAD17vLVDgEPYmFja2dyb3VuZENvbG9y"
+        "AXcHZGVmYXVsdIf17vLVDgEDDmJsb2NrQ29udGFpbmVyBwD17vLVDhADDmJ1bGxldExpc3RJdGVt"
+        "BwD17vLVDhEGBAD17vLVDhIBd4b17vLVDhMEYm9sZAJ7fYT17vLVDhQCb3KG9e7y1Q4WBGJvbGQE"
+        "bnVsbIT17vLVDhcCbGQoAPXu8tUOEQ10ZXh0QWxpZ25tZW50AXcEbGVmdCgA9e7y1Q4QAmlkAXck"
+        "ZDM1MWUwNjgtM2U1NS00MjI2LThlYTUtYWJiMjYzMTk4ZTJhKAD17vLVDhAJdGV4dENvbG9yAXcH"
+        "ZGVmYXVsdCgA9e7y1Q4QD2JhY2tncm91bmRDb2xvcgF3B2RlZmF1bHSH9e7y1Q4QAw5ibG9ja0Nv"
+        "bnRhaW5lcgcA9e7y1Q4eAwlwYXJhZ3JhcGgoAPXu8tUOHw10ZXh0QWxpZ25tZW50AXcEbGVmdCgA"
+        "9e7y1Q4eAmlkAXckODk3MDBjMDctZTBlMS00ZmUwLWFjYTItODQ5MzIwOWE3ZTQyKAD17vLVDh4J"
+        "dGV4dENvbG9yAXcHZGVmYXVsdCgA9e7y1Q4eD2JhY2tncm91bmRDb2xvcgF3B2RlZmF1bHQA"
+    )
+
+    assert yjs_base64_to_text(base64_string) == "Hello world"

src/backend/core/utils.py

@@ -2,6 +2,7 @@
 Utilities for the core app.
 """
 
+import base64
 import smtplib
 from logging import getLogger
 
@@ -12,6 +13,9 @@ from django.template.loader import render_to_string
 from django.utils.translation import gettext_lazy as _
 from django.utils.translation import override
 
+import y_py as Y
+from bs4 import BeautifulSoup
+
 logger = getLogger(__name__)
 
 
@@ -38,3 +42,17 @@ def email_invitation(language, email, document_id):
 
     except smtplib.SMTPException as exception:
         logger.error("invitation to %s was not sent: %s", email, exception)
+
+
+def yjs_base64_to_text(base64_string):
+    """Extract text from base64 yjs document"""
+
+    decoded_bytes = base64.b64decode(base64_string)
+    uint8_array = bytearray(decoded_bytes)
+
+    doc = Y.YDoc()  # pylint: disable=E1101
+    Y.apply_update(doc, uint8_array)  # pylint: disable=E1101
+    blocknote_structure = str(doc.get_xml_element("document-store"))
+
+    soup = BeautifulSoup(blocknote_structure, "html.parser")
+    return soup.get_text(separator=" ").strip()

src/backend/pyproject.toml

@@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "impress"
-version = "1.3.0"
+version = "1.4.0"
 authors = [{ "name" = "DINUM", "email" = "dev@mail.numerique.gouv.fr" }]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -25,6 +25,7 @@ license = { file = "LICENSE" }
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
+    "beautifulsoup4==4.12.3",
     "boto3==1.35.10",
     "Brotli==1.1.0",
     "celery[redis]==5.4.0",
@@ -57,6 +58,7 @@ dependencies = [
     "WeasyPrint>=60.2",
     "whitenoise==6.7.0",
     "mozilla-django-oidc==4.0.1",
+    "y-py==0.5.5",
 ]
 
 [project.urls]

src/frontend/apps/e2e/__tests__/app-impress/doc-table-content.spec.ts

@@ -20,7 +20,7 @@ test.describe('Doc Table Content', () => {
     await page.getByLabel('Open the document options').click();
     await page
       .getByRole('button', {
-        name: 'Table of content',
+        name: 'Table of contents',
       })
       .click();
 
@@ -30,6 +30,8 @@ test.describe('Doc Table Content', () => {
     await editor.locator('.bn-block-outer').last().fill('/');
     await page.getByText('Heading 1').click();
     await page.keyboard.type('Hello World');
+    await editor.getByText('Hello').dblclick();
+    await page.getByRole('button', { name: 'Strike' }).click();
 
     await page.locator('.bn-block-outer').last().click();
 
@@ -40,7 +42,7 @@ test.describe('Doc Table Content', () => {
 
     await editor.locator('.bn-block-outer').last().fill('/');
     await page.getByText('Heading 2').click();
-    await page.keyboard.type('Super World');
+    await page.keyboard.type('Super World', { delay: 100 });
 
     await page.locator('.bn-block-outer').last().click();
 
@@ -58,15 +60,15 @@ test.describe('Doc Table Content', () => {
     const another = panel.getByText('Another World');
 
     await expect(hello).toBeVisible();
-    await expect(hello).toHaveCSS('font-size', '19.2px');
+    await expect(hello).toHaveCSS('font-size', /19/);
     await expect(hello).toHaveAttribute('aria-selected', 'true');
 
     await expect(superW).toBeVisible();
-    await expect(superW).toHaveCSS('font-size', '16px');
+    await expect(superW).toHaveCSS('font-size', /16/);
     await expect(superW).toHaveAttribute('aria-selected', 'false');
 
     await expect(another).toBeVisible();
-    await expect(another).toHaveCSS('font-size', '12.8px');
+    await expect(another).toHaveCSS('font-size', /12/);
     await expect(another).toHaveAttribute('aria-selected', 'false');
 
     await hello.click();

src/frontend/apps/e2e/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app-e2e",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "private": true,
   "scripts": {
     "lint": "eslint . --ext .ts",

src/frontend/apps/impress/package.json

@@ -1,6 +1,6 @@
 {
   "name": "app-impress",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "private": true,
   "scripts": {
     "dev": "next dev",
@@ -33,8 +33,8 @@
     "react-i18next": "15.0.2",
     "react-select": "5.8.0",
     "styled-components": "6.1.13",
-    "yjs": "*",
     "y-protocols": "1.0.6",
+    "yjs": "*",
     "zustand": "4.5.5"
   },
   "devDependencies": {

src/frontend/apps/impress/src/components/Panel.tsx

@@ -36,10 +36,10 @@ export const Panel = ({
       $width="100%"
       $maxWidth="20rem"
       $position="sticky"
-      $maxHeight="96vh"
+      $maxHeight="99vh"
       $height="100%"
       $css={`
-        top: 2vh;
+        top: 0vh;
         transition: ${transition};
         ${
           !isOpen &&

src/frontend/apps/impress/src/features/docs/doc-header/components/DocToolBox.tsx

@@ -91,7 +91,7 @@ export const DocToolBox = ({ doc }: DocToolBoxProps) => {
             icon={<span className="material-icons">summarize</span>}
             size="small"
           >
-            <Text $theme="primary">{t('Table of content')}</Text>
+            <Text $theme="primary">{t('Table of contents')}</Text>
           </Button>
           <Button
             onClick={() => {

src/frontend/apps/impress/src/features/docs/doc-table-content/components/Heading.tsx

@@ -46,6 +46,7 @@ export const Heading = ({
           block: 'start',
         });
       }}
+      $css="text-align: left;"
     >
       <Text
         $theme="primary"

src/frontend/apps/impress/src/features/docs/doc-table-content/components/TableContent.tsx

@@ -10,11 +10,28 @@ import { useDocTableContentStore } from '../stores';
 
 import { Heading } from './Heading';
 
+const recursiveTextContent = (content: HeadingBlock['content']): string => {
+  if (!content) {
+    return '';
+  }
+
+  return content.reduce((acc, content) => {
+    if (content.type === 'text') {
+      return acc + content.text;
+    } else if (content.type === 'link') {
+      return acc + recursiveTextContent(content.content);
+    }
+
+    return acc;
+  }, '');
+};
+
 type HeadingBlock = {
   id: string;
   type: string;
   text: string;
   content: HeadingBlock[];
+  contentText: string;
   props: {
     level: number;
   };
@@ -31,9 +48,14 @@ export const TableContent = ({ doc }: TableContentProps) => {
   const editor = docsStore?.[doc.id]?.editor;
   const headingFiltering = useCallback(
     () =>
-      editor?.document.filter(
-        (block) => block.type === 'heading',
-      ) as unknown as HeadingBlock[],
+      editor?.document
+        .filter((block) => block.type === 'heading')
+        .map((block) => ({
+          ...block,
+          contentText: recursiveTextContent(
+            block.content as unknown as HeadingBlock['content'],
+          ),
+        })) as unknown as HeadingBlock[],
     [editor?.document],
   );
 
@@ -71,7 +93,7 @@ export const TableContent = ({ doc }: TableContentProps) => {
 
       for (const heading of headings) {
         const elHeading = document.body.querySelector(
-          `.bn-block-outer[data-id="${heading.id}"]`,
+          `.bn-block-outer[data-id="${heading.id}"] [data-content-type="heading"]:first-child`,
         );
 
         if (!elHeading) {
@@ -121,21 +143,16 @@ export const TableContent = ({ doc }: TableContentProps) => {
     <Panel setIsPanelOpen={setClosePanel}>
       <Box $padding="small" $maxHeight="95%">
         <Box $overflow="auto">
-          {headings?.map((heading) => {
-            const content = heading.content?.[0];
-            const text = content?.type === 'text' ? content.text : '';
-
-            return (
-              <Heading
-                editor={editor}
-                headingId={heading.id}
-                level={heading.props.level}
-                text={text}
-                key={heading.id}
-                isHighlight={headingIdHighlight === heading.id}
-              />
-            );
-          })}
+          {headings?.map((heading) => (
+            <Heading
+              editor={editor}
+              headingId={heading.id}
+              level={heading.props.level}
+              text={heading.contentText}
+              key={heading.id}
+              isHighlight={headingIdHighlight === heading.id}
+            />
+          ))}
         </Box>
         <Box
           $height="1px"

src/frontend/apps/impress/src/i18n/translations.json

@@ -20,6 +20,7 @@
       "Content modal to delete document": "Contenu modal pour supprimer le document",
       "Content modal to export the document": "Contenu modal pour exporter le document",
       "Cookies placed": "Cookies déposés",
+      "Copy link": "Copier le lien",
       "Copyright": "Copyright",
       "Create a new document": "Créer un nouveau document",
       "Create the document": "Créer le document",
@@ -29,9 +30,10 @@
       "Defender of Rights - Free response - 71120 75342 Paris CEDEX 07": "Défenseur des droits - Réponse gratuite - 71120 75342 Paris CEDEX 07",
       "Delete document": "Supprimer le document",
       "Deleting the document \"{{title}}\"": "Suppression du document \"{{title}}\"",
+      "Doc visibility card": "Carte de visibilité du doc",
       "Docs": "Docs",
-      "Docs Description": "Description de Docs",
       "Docs Logo": "Logo Docs",
+      "Docs: Your new companion to collaborate on documents efficiently, intuitively, and securely.": "Docs : Votre nouveau compagnon pour collaborer sur des documents efficacement, intuitivement et en toute sécurité.",
       "Document icon": "Icône du document",
       "Document name": "Nom du document",
       "Document panel": "Panneau du document",
@@ -45,6 +47,7 @@
       "Export": "Exporter",
       "Export your document, it will be inserted in the selected template.": "Exportez votre document, il sera inséré dans le modèle sélectionné.",
       "Failed to add the member in the document.": "Impossible d'ajouter le membre dans le document.",
+      "Failed to copy link": "Échec de la copie du lien",
       "Failed to create the invitation for {{email}}.": "Impossible de créer l'invitation pour {{email}}.",
       "Find a member to add to the document": "Trouver un membre à ajouter au document",
       "French Interministerial Directorate for Digital Affairs (DINUM), 20 avenue de Ségur 75007 Paris.": "Direction interministérielle des affaires numériques (DINUM), 20 avenue de Segur 75007 Paris.",
@@ -56,7 +59,6 @@
       "Invitation sent to {{email}}.": "Invitation envoyée à {{email}}.",
       "Invite new members to {{title}}": "Invitez de nouveaux membres à rejoindre {{title}}",
       "Invited": "Invité",
-      "Is it public ?": "Est-ce public?",
       "It is the card information about the document.": "Il s'agit de la carte d'information du document.",
       "It seems that the page you are looking for does not exist or cannot be displayed correctly.": "Il semble que la page que vous cherchez n'existe pas ou ne puisse pas être affichée correctement.",
       "It's true, you didn't have to click on a block that covers half the page to say you agree to the placement of cookies — even if you don't know what it means!": "C'est vrai, vous n'avez pas à cliquer sur un bloc qui couvre la moitié de la page pour dire que vous acceptez le placement de cookies — même si vous ne savez pas ce que cela signifie !",
@@ -64,13 +66,14 @@
       "Language Icon": "Icône de langue",
       "Legal Notice": "Mentions Legales",
       "Legal notice": "Mention légale",
+      "Link Copied !": "Lien copié !",
       "List invitation card": "Carte de liste d'invitation",
       "List members card": "Carte liste des membres",
+      "Login": "Connexion",
       "Logout": "Se déconnecter",
       "Members": "Membres",
       "Modal confirmation to restore the version": "Modale de confirmation pour restaurer la version",
       "More info?": "Plus d'infos ?",
-      "My account": "Mon compte",
       "No editor found": "Pas d'éditeur trouvé",
       "Nothing exceptional, no special privileges related to a .gouv.fr.": "Rien d'exceptionnel, pas de privilèges spéciaux liés à un .gouv.fr.",
       "Offline ?!": "Hors-ligne ?!",
@@ -94,16 +97,16 @@
       "Restore this version": "Restaurer cette version",
       "Restore this version?": "Restaurer cette version?",
       "Role": "Rôle",
-      "SUMMARY": "RÉSUMÉ",
       "Search by email": "Recherche par email",
       "Send a letter by post (free of charge, no stamp needed):": "Envoyer un courrier par la poste (gratuit, ne pas mettre de timbre):",
       "Share": "Partager",
       "Something bad happens, please retry.": "Une erreur inattendue s'est produite, veuillez réessayer.",
       "Stéphanie Schaer: Interministerial Digital Director (DINUM).": "Stéphanie Schaer: Directrice numérique interministériel (DINUM).",
-      "Summary": "Résumé",
+      "Table of contents": "Table des matières",
       "Template": "Template",
       "The document has been deleted.": "Le document a bien été supprimé.",
       "The document has been updated.": "Le document a été mis à jour.",
+      "The document visiblitity has been updated.": "La visibilité du document a été mise à jour.",
       "The invitation has been removed.": "L'invitation a été supprimée.",
       "The member has been removed from the document": "Le membre a été retiré du document",
       "The role has been updated": "Le rôle a été mis à jour",

src/frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "impress",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "private": true,
   "workspaces": {
     "packages": [

src/frontend/packages/eslint-config-impress/package.json

@@ -1,6 +1,6 @@
 {
   "name": "eslint-config-impress",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "license": "MIT",
   "scripts": {
     "lint": "eslint --ext .js ."

src/frontend/packages/i18n/package.json

@@ -1,6 +1,6 @@
 {
   "name": "packages-i18n",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "private": true,
   "scripts": {
     "extract-translation": "yarn extract-translation:impress",

src/frontend/servers/y-provider/package.json

@@ -1,6 +1,6 @@
 {
   "name": "server-y-provider",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Y.js provider for docs",
   "repository": "https://github.com/numerique-gouv/impress",
   "license": "MIT",

src/helm/env.d/preprod/values.impress.yaml.gotmpl

@@ -1,7 +1,7 @@
 image:
   repository: lasuite/impress-backend
   pullPolicy: Always
-  tag: "v1.3.0-preprod"
+  tag: "v1.4.0-preprod"
 
 backend:
   migrateJobAnnotations:
@@ -124,13 +124,13 @@ frontend:
   image:
     repository: lasuite/impress-frontend
     pullPolicy: Always
-    tag: "v1.3.0-preprod"
+    tag: "v1.4.0-preprod"
 
 yProvider:
   image:
     repository: lasuite/impress-y-provider
     pullPolicy: Always
-    tag: "v1.3.0-preprod"
+    tag: "v1.4.0-preprod"
 
 ingress:
   enabled: true

src/helm/env.d/production/values.impress.yaml.gotmpl

@@ -1,7 +1,7 @@
 image:
   repository: lasuite/impress-backend
   pullPolicy: Always
-  tag: "v1.3.0"
+  tag: "v1.4.0"
 
 backend:
   migrateJobAnnotations:
@@ -124,13 +124,13 @@ frontend:
   image:
     repository: lasuite/impress-frontend
     pullPolicy: Always
-    tag: "v1.3.0"
+    tag: "v1.4.0"
 
 yProvider:
   image:
     repository: lasuite/impress-y-provider
     pullPolicy: Always
-    tag: "v1.3.0"
+    tag: "v1.4.0"
 
 ingress:
   enabled: true

src/mail/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mail_mjml",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "An util to generate html and text django's templates from mjml templates",
   "type": "module",
   "dependencies": {