✨(backend) use docling

I save my changes but this is deprecated as we want must use docling-serve.
2026-01-08 17:08:16 +01:00
13 changed files with 290 additions and 81 deletions
@@ -82,3 +82,6 @@ db.sqlite3

 # Docker compose override
 compose.override.yml
+
+# Docling
+docling-models
@@ -7,8 +7,16 @@ from urllib.parse import urljoin
 from django.conf import settings

 import requests
+from docling.backend.pypdfium2_backend import PyPdfiumDocumentBackend
+from docling.datamodel.base_models import InputFormat
+from docling.datamodel.pipeline_options import PdfPipelineOptions, TableStructureOptions
+from docling.document_converter import DocumentConverter as DoclingDocumentConverter
+from docling.document_converter import PdfFormatOption
+from docling_core.types.io import DocumentStream

-from chat.agent_rag.document_converter.markitdown import DocumentConverter
+from chat.agent_rag.document_converter.markitdown import (
+    DocumentConverter as MarkitdownDocumentConverter,
+)

 logger = logging.getLogger(__name__)

@@ -61,6 +69,32 @@ class AlbertParser(BaseParser):
        """Parse document based on content type."""
        if content_type == "application/pdf":
            return self.parse_pdf_document(name=name, content_type=content_type, content=content)
-        return DocumentConverter().convert_raw(
+        return MarkitdownDocumentConverter().convert_raw(
            name=name, content_type=content_type, content=content
        )
+
+
+class DoclingParser(BaseParser):
+    """Document parser using Docling's DocumentConverter."""
+
+    artifacts_path = "src/backend/docling-models"
+
+    def __init__(self):
+        pipeline_options = PdfPipelineOptions(artifacts_path=self.artifacts_path)
+        pipeline_options.do_ocr = True
+        pipeline_options.do_table_structure = True
+        pipeline_options.table_structure_options = TableStructureOptions(do_cell_matching=False)
+
+        self.converter = DoclingDocumentConverter(
+            format_options={
+                InputFormat.PDF: PdfFormatOption(
+                    pipeline_options=pipeline_options,
+                    backend=PyPdfiumDocumentBackend
+                )}
+        )
+
+    def parse_document(self, name: str, content_type: str, content: bytes) -> str:
+        """Parse document using Docling's DocumentConverter."""
+        return self.converter.convert(
+            DocumentStream(name=name, stream=BytesIO(content))
+        ).document.export_to_markdown()
@@ -13,7 +13,7 @@ import requests

 from chat.agent_rag.albert_api_constants import Searches
 from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
-from chat.agent_rag.document_converter.parser import AlbertParser
+from chat.agent_rag.document_converter.parser import DoclingParser
 from chat.agent_rag.document_rag_backends.base_rag_backend import BaseRagBackend

 logger = logging.getLogger(__name__)
@@ -45,7 +45,7 @@ class AlbertRagBackend(BaseRagBackend):  # pylint: disable=too-many-instance-att
        self._documents_endpoint = urljoin(self._base_url, "/v1/documents")
        self._search_endpoint = urljoin(self._base_url, "/v1/search")
        self._default_collection_description = "Temporary collection for RAG document search"
-        self.parser = AlbertParser()
+        self.parser = DoclingParser()

    def create_collection(self, name: str, description: Optional[str] = None) -> str:
        """
@@ -12,7 +12,7 @@ from django.utils import timezone
 import requests

 from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
-from chat.agent_rag.document_converter.parser import AlbertParser
+from chat.agent_rag.document_converter.parser import DoclingParser
 from chat.agent_rag.document_rag_backends.base_rag_backend import BaseRagBackend
 from utils.oidc import with_fresh_access_token

@@ -42,7 +42,7 @@ class FindRagBackend(BaseRagBackend):
        self.api_key = settings.FIND_API_KEY
        self.search_endpoint = "api/v1.0/documents/search/"
        self.indexing_endpoint = "api/v1.0/documents/index/"
-        self.parser = AlbertParser()  # Find Rag relies on Albert parser
+        self.parser = DoclingParser()

    def create_collection(self, name: str, description: Optional[str] = None) -> str:
        """
@@ -11,7 +11,7 @@ import requests

 from chat.agent_rag.albert_api_constants import Searches
 from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
-from chat.agent_rag.document_converter.markitdown import DocumentConverter
+from chat.agent_rag.document_converter.parser import DoclingParser
 from chat.models import ChatConversation

 logger = logging.getLogger(__name__)
@@ -80,58 +80,6 @@ class AlbertRagDocumentSearch:
        self.conversation.collection_id = str(response.json()["id"])
        return True

-    def _parse_pdf_document(self, name: str, content_type: str, content: BytesIO) -> str:
-        """
-        Parse the PDF document content and return the text content.
-        This method should handle the logic to convert the PDF into
-        a format suitable for the Albert API.
-        """
-        response = requests.post(
-            self._pdf_parser_endpoint,
-            headers=self._headers,
-            files={
-                "file": (
-                    name,
-                    content,
-                    content_type,
-                ),  # Use the name as the filename in the request
-                "output_format": (None, "markdown"),  # Specify the output format as Markdown,
-            },
-            timeout=settings.ALBERT_API_PARSE_TIMEOUT,
-        )
-        response.raise_for_status()
-
-        return "\n\n".join(
-            document_page["content"] for document_page in response.json().get("data", [])
-        )
-
-    def parse_document(self, name: str, content_type: str, content: BytesIO):
-        """
-        Parse the document and prepare it for the search operation.
-        This method should handle the logic to convert the document
-        into a format suitable for the Albert API.
-
-        Args:
-            name (str): The name of the document.
-            content_type (str): The MIME type of the document (e.g., "application/pdf").
-            content (BytesIO): The content of the document as a BytesIO stream.
-
-        Returns:
-            str: The document content in Markdown format.
-        """
-        # Implement the parsing logic here
-        if content_type == "application/pdf":
-            # Handle PDF parsing
-            markdown_content = self._parse_pdf_document(
-                name=name, content_type=content_type, content=content
-            )
-        else:
-            markdown_content = DocumentConverter().convert_raw(
-                name=name, content_type=content_type, content=content
-            )
-
-        return markdown_content
-
    def _store_document(self, name: str, content: str):
        """
        Store the document content in the Albert collection.
@@ -156,7 +104,7 @@ class AlbertRagDocumentSearch:
        logger.debug(response.json())
        response.raise_for_status()

-    def parse_and_store_document(self, name: str, content_type: str, content: BytesIO):
+    def parse_and_store_document(self, name: str, content_type: str, content: bytes):
        """
        Parse the document and store it in the Albert collection.

@@ -165,7 +113,9 @@ class AlbertRagDocumentSearch:
            content_type (str): The MIME type of the document (e.g., "application/pdf").
            content (BytesIO): The content of the document as a BytesIO stream.
        """
-        document_content = self.parse_document(name, content_type, content)
+        document_content = DoclingParser().parse_document(
+            name=name, content_type=content_type, content=content
+        )
        self._store_document(name, document_content)
        return document_content

@@ -0,0 +1,29 @@
+"""
+Unit tests for the DocumentConverter.
+
+Only for coverage as the DocumentConverter is a simple wrapper around MarkItDown.
+"""
+
+from io import BytesIO
+
+from docling.document_converter import DocumentConverter
+from docling_core.types.io import DocumentStream
+
+
+def main():
+    """Test that the DocumentConverter calls the underlying MarkItDown converter."""
+    file_path = "test.pdf"
+    converter = DocumentConverter()
+
+    # Convert from file content instead of file path
+    with open(file_path, "rb") as file:
+        content = file.read()
+        stream = DocumentStream(name="test.pdf", stream=BytesIO(content))
+        result = converter.convert(stream)
+        markdown = result.document.export_to_markdown()
+
+    assert markdown == "Document PDF test"
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,90 @@
+%PDF-1.4
+%Çì�¢
+5 0 obj
+<</Length 6 0 R/Filter /FlateDecode>>
+stream
+xœMŽ±NÄ0†÷<…Çd¨ÏNœ¸^OÀÀ§l'¦Šc*¨âx’RªÚƒÿóo{BŽ@=ÿ›ivnq#¦«xì§ÎÕ�.
+ÍQoŽÐÌ„WÆ „#h!¨³»ú‡À˜5Sò_a Œ&¦â§�°•Ÿƒ4‡!¢ÊÅ¿ÿÑÏ½wÊ%çÑC—Y4[ò/a�ö³n‡D¢
+‹æhû¨Z<nØö‡�1F3Ýaj–·úì«{mùµi:uendstream
+endobj
+6 0 obj
+180
+endobj
+4 0 obj
+<</Type/Page/MediaBox [0 0 595 842]
+/Rotate 0/Parent 3 0 R
+/Resources<</ProcSet[/PDF /Text]
+/Font 8 0 R
+>>
+/Contents 5 0 R
+>>
+endobj
+3 0 obj
+<< /Type /Pages /Kids [
+4 0 R
+] /Count 1
+>>
+endobj
+1 0 obj
+<</Type /Catalog /Pages 3 0 R
+/Metadata 9 0 R
+>>
+endobj
+8 0 obj
+<</R7
+7 0 R>>
+endobj
+7 0 obj
+<</BaseFont/Times-Roman/Type/Font
+/Subtype/Type1>>
+endobj
+9 0 obj
+<</Type/Metadata
+/Subtype/XML/Length 1549>>stream
+<?xpacket begin='ï»¿' id='W5M0MpCehiHzreSzNTczkc9d'?>
+<?adobe-xap-filters esc="CRLF"?>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
+<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:pdf='http://ns.adobe.com/pdf/1.3/'><pdf:Producer>GPL Ghostscript 9.06</pdf:Producer>
+<pdf:Keywords>()</pdf:Keywords>
+</rdf:Description>
+<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:xmp='http://ns.adobe.com/xap/1.0/'><xmp:ModifyDate>2014-12-22T00:49:20+01:00</xmp:ModifyDate>
+<xmp:CreateDate>2014-12-22T00:49:20+01:00</xmp:CreateDate>
+<xmp:CreatorTool>PDFCreator Version 1.6.0</xmp:CreatorTool></rdf:Description>
+<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c'/>
+<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>test_word</rdf:li></rdf:Alt></dc:title><dc:creator><rdf:Seq><rdf:li>Seb</rdf:li></rdf:Seq></dc:creator><dc:description><rdf:Seq><rdf:li>()</rdf:li></rdf:Seq></dc:description></rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+                                                                        
+                                                                        
+<?xpacket end='w'?>
+endstream
+endobj
+2 0 obj
+<</Producer(GPL Ghostscript 9.06)
+/CreationDate(D:20141222004920+01'00')
+/ModDate(D:20141222004920+01'00')
+/Title(\376\377\000t\000e\000s\000t\000_\000w\000o\000r\000d)
+/Creator(\376\377\000P\000D\000F\000C\000r\000e\000a\000t\000o\000r\000 \000V\000e\000r\000s\000i\000o\000n\000 \0001\000.\0006\000.\0000)
+/Author(\376\377\000S\000e\000b)
+/Keywords()
+/Subject()>>endobj
+xref
+0 10
+0000000000 65535 f 
+0000000484 00000 n 
+0000002268 00000 n 
+0000000425 00000 n 
+0000000284 00000 n 
+0000000015 00000 n 
+0000000265 00000 n 
+0000000577 00000 n 
+0000000548 00000 n 
+0000000643 00000 n 
+trailer
+<< /Size 10 /Root 1 0 R /Info 2 0 R
+/ID [<0CB231047435B33BCE0B1C6881DCF011><0CB231047435B33BCE0B1C6881DCF011>]
+>>
+startxref
+2648
+%%EOF
@@ -0,0 +1,18 @@
+"""
+Unit tests for the DoclingParser.
+"""
+from chat.agent_rag.document_converter.parser import DoclingParser
+
+
+def test_document_converter():
+    """Test that the DocumentConverter calls the underlying MarkItDown converter."""
+    file_name = "test"
+    content_type = "application/pdf"
+    file_path = "src/backend/chat/tests/data/test.pdf"
+    parser = DoclingParser()
+
+    with open(file_path, "rb") as file:
+        content = file.read()
+        result = parser.parse_document(name= file_name, content_type= content_type, content= content)
+
+    assert "Document PDF test" in result
@@ -5,28 +5,21 @@ Only for coverage as the DocumentConverter is a simple wrapper around MarkItDown
 """

 from io import BytesIO
-from unittest.mock import MagicMock, patch

 from chat.agent_rag.document_converter.markitdown import DocumentConverter


-@patch("chat.agent_rag.document_converter.markitdown.MarkItDown")
-def test_document_converter(mock_markitdown: MagicMock):
+def test_document_converter():
    """Test that the DocumentConverter calls the underlying MarkItDown converter."""
-    mock_conversion = MagicMock()
-    mock_conversion.text_content = "converted text"
-    mock_markitdown.return_value.convert_stream.return_value = mock_conversion
-
+    file_path = "src/backend/chat/tests/data/test.pdf"
    converter = DocumentConverter()

-    result = converter.convert_raw(
-        name="test.pdf",
-        content_type="application/pdf",
-        content=b"test content",
-    )
+    with open(file_path, "rb") as file:
+        content = file.read()
+        result = converter.convert_raw(
+            name="test.pdf",
+            content_type="application/pdf",
+            content=content,
+        )

-    assert result == "converted text"
-    converter.converter.convert_stream.assert_called_once()  # pylint: disable=no-member
-    args, kwargs = converter.converter.convert_stream.call_args  # pylint: disable=no-member
-    assert isinstance(args[0], BytesIO)
-    assert kwargs["file_extension"] == ".pdf"
+    assert result == "Document PDF test\n\n"
@@ -0,0 +1,90 @@
+%PDF-1.4
+%Çì�¢
+5 0 obj
+<</Length 6 0 R/Filter /FlateDecode>>
+stream
+xœMŽ±NÄ0†÷<…Çd¨ÏNœ¸^OÀÀ§l'¦Šc*¨âx’RªÚƒÿóo{BŽ@=ÿ›ivnq#¦«xì§ÎÕ�.
+ÍQoŽÐÌ„WÆ „#h!¨³»ú‡À˜5Sò_a Œ&¦â§�°•Ÿƒ4‡!¢ÊÅ¿ÿÑÏ½wÊ%çÑC—Y4[ò/a�ö³n‡D¢
+‹æhû¨Z<nØö‡�1F3Ýaj–·úì«{mùµi:uendstream
+endobj
+6 0 obj
+180
+endobj
+4 0 obj
+<</Type/Page/MediaBox [0 0 595 842]
+/Rotate 0/Parent 3 0 R
+/Resources<</ProcSet[/PDF /Text]
+/Font 8 0 R
+>>
+/Contents 5 0 R
+>>
+endobj
+3 0 obj
+<< /Type /Pages /Kids [
+4 0 R
+] /Count 1
+>>
+endobj
+1 0 obj
+<</Type /Catalog /Pages 3 0 R
+/Metadata 9 0 R
+>>
+endobj
+8 0 obj
+<</R7
+7 0 R>>
+endobj
+7 0 obj
+<</BaseFont/Times-Roman/Type/Font
+/Subtype/Type1>>
+endobj
+9 0 obj
+<</Type/Metadata
+/Subtype/XML/Length 1549>>stream
+<?xpacket begin='ï»¿' id='W5M0MpCehiHzreSzNTczkc9d'?>
+<?adobe-xap-filters esc="CRLF"?>
+<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
+<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
+<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:pdf='http://ns.adobe.com/pdf/1.3/'><pdf:Producer>GPL Ghostscript 9.06</pdf:Producer>
+<pdf:Keywords>()</pdf:Keywords>
+</rdf:Description>
+<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:xmp='http://ns.adobe.com/xap/1.0/'><xmp:ModifyDate>2014-12-22T00:49:20+01:00</xmp:ModifyDate>
+<xmp:CreateDate>2014-12-22T00:49:20+01:00</xmp:CreateDate>
+<xmp:CreatorTool>PDFCreator Version 1.6.0</xmp:CreatorTool></rdf:Description>
+<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c'/>
+<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>test_word</rdf:li></rdf:Alt></dc:title><dc:creator><rdf:Seq><rdf:li>Seb</rdf:li></rdf:Seq></dc:creator><dc:description><rdf:Seq><rdf:li>()</rdf:li></rdf:Seq></dc:description></rdf:Description>
+</rdf:RDF>
+</x:xmpmeta>
+                                                                        
+                                                                        
+<?xpacket end='w'?>
+endstream
+endobj
+2 0 obj
+<</Producer(GPL Ghostscript 9.06)
+/CreationDate(D:20141222004920+01'00')
+/ModDate(D:20141222004920+01'00')
+/Title(\376\377\000t\000e\000s\000t\000_\000w\000o\000r\000d)
+/Creator(\376\377\000P\000D\000F\000C\000r\000e\000a\000t\000o\000r\000 \000V\000e\000r\000s\000i\000o\000n\000 \0001\000.\0006\000.\0000)
+/Author(\376\377\000S\000e\000b)
+/Keywords()
+/Subject()>>endobj
+xref
+0 10
+0000000000 65535 f 
+0000000484 00000 n 
+0000002268 00000 n 
+0000000425 00000 n 
+0000000284 00000 n 
+0000000015 00000 n 
+0000000265 00000 n 
+0000000577 00000 n 
+0000000548 00000 n 
+0000000643 00000 n 
+trailer
+<< /Size 10 /Root 1 0 R /Info 2 0 R
+/ID [<0CB231047435B33BCE0B1C6881DCF011><0CB231047435B33BCE0B1C6881DCF011>]
+>>
+startxref
+2648
+%%EOF
@@ -121,7 +121,7 @@ def fixture_mock_document_api():
        status=status.HTTP_200_OK,
    )

-    # Mock PDF parsing
+    # Mock Albert PDF parsing -> deprecated
    responses.post(
        "https://albert.api.etalab.gouv.fr/v1/parse-beta",
        json={
@@ -675,7 +675,7 @@ def test_post_conversation_with_document_upload_summarize(  # pylint: disable=to
        'document discusses various topics."}\n'
        '0:"The document discusses various topics."\n'
        'f:{"messageId":"<mocked_uuid>"}\n'
-        'd:{"finishReason":"stop","usage":{"promptTokens":287,"completionTokens":19}}\n'
+        'd:{"finishReason":"stop","usage":{"promptTokens":283,"completionTokens":19}}\n'
    )

    # Check that the conversation was updated
@@ -101,7 +101,7 @@ async def document_summarize(  # pylint: disable=too-many-locals
        )
        documents_chunks = chunker(
            [doc[1] for doc in documents],
-            overlap=settings.SUMMARIZATION_OVERLAP_SIZE,
+            # overlap=settings.SUMMARIZATION_OVERLAP_SIZE,
        )

        logger.info(
@@ -43,7 +43,9 @@ dependencies = [
    "djangorestframework==3.16.1",
    "drf_spectacular==0.29.0",
    "dockerflow==2024.4.2",
+    "docling",
    "easy_thumbnails==2.10.1",
+    "easyocr",
    "factory_boy==3.3.3",
    "gunicorn==23.0.0",
    "jsonschema==4.25.1",