electron-rare/conversations
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

✨(backend) implement DoclingServeParser

Browse Source 
change parsers and update Docker configuration
This commit is contained in:
charles
2026-01-14 16:25:02 +01:00
parent 12e6be02c9
commit 17e49b5088
7 changed files with 58 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compose.yml
+9
View File
@@ -16,6 +16,13 @@ services:
redis:
image: redis:5
docling-serve:
image: quay.io/docling-project/docling-serve:latest
ports:
- "5001:5001"
environment:
- DOCLING_SERVE_ARTIFACTS_PATH=""
maildev:
image: maildev/maildev:latest
ports:
@@ -87,6 +94,8 @@ services:
condition: service_started
createbuckets:
condition: service_started
docling-serve:
condition: service_started
nginx:
image: nginx:1.25
docs/env.md
+2 -1
View File
@@ -98,7 +98,8 @@ These are the environment variables you can set for the `conversations-backend`
| FIND_API_KEY | API key of Find | |
| FIND_API_URL | URL of Find | `https://app-find/api` |
| FIND_API_TIMEOUT | Find API timeout | 30 |
| DOCLING_SERVE_URL | URL of Docling Serve | `http://docling-serve:5001` |
| DOCLING_API_TIMEOUT | Docling API timeout | 60 |
## conversations-frontend image
src/backend/chat/agent_rag/document_converter/parser.py
+27
View File
@@ -33,6 +33,33 @@ class BaseParser:
raise NotImplementedError("Must be implemented in subclass.")
class DoclingServeParser(BaseParser):
"""Document parser using Docling Serve API."""
def __init__(self):
self.endpoint = urljoin(settings.DOCLING_SERVE_URL, "/v1/convert/file")
def parse_document(self, name: str, content_type: str, content: bytes) -> str:
"""Parse document using Docling Serve API."""
timeout = settings.DOCLING_SERVE_TIMEOUT
response = requests.post(
self.endpoint,
files={
"files": content,
},
data={
"image_export_mode": "placeholder",
"md_page_break_placeholder": "\n\n",
"do_picture_description": "true",
"document_timeout": timeout,
},
timeout=timeout,
)
response.raise_for_status()
return response.json()["document"]["md_content"]
class AlbertParser(BaseParser):
"""Document parser using Albert API for PDFs and DocumentConverter for other formats."""
src/backend/chat/agent_rag/document_rag_backends/albert_rag_backend.py
+2 -2
View File
@@ -13,7 +13,7 @@ import requests
from chat.agent_rag.albert_api_constants import Searches
from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
from chat.agent_rag.document_converter.parser import AlbertParser
from chat.agent_rag.document_converter.parser import DoclingServeParser
from chat.agent_rag.document_rag_backends.base_rag_backend import BaseRagBackend
logger = logging.getLogger(__name__)
@@ -45,7 +45,7 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
self._documents_endpoint = urljoin(self._base_url, "/v1/documents")
self._search_endpoint = urljoin(self._base_url, "/v1/search")
self._default_collection_description = "Temporary collection for RAG document search"
self.parser = AlbertParser()
self.parser = DoclingServeParser()
def create_collection(self, name: str, description: Optional[str] = None) -> str:
"""
src/backend/chat/agent_rag/document_rag_backends/find_rag_backend.py
+2 -2
View File
@@ -12,7 +12,7 @@ from django.utils import timezone
import requests
from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
from chat.agent_rag.document_converter.parser import AlbertParser
from chat.agent_rag.document_converter.parser import DoclingServeParser
from chat.agent_rag.document_rag_backends.base_rag_backend import BaseRagBackend
from utils.oidc import with_fresh_access_token
@@ -42,7 +42,7 @@ class FindRagBackend(BaseRagBackend):
self.api_key = settings.FIND_API_KEY
self.search_endpoint = "api/v1.0/documents/search/"
self.indexing_endpoint = "api/v1.0/documents/index/"
self.parser = AlbertParser() # Find Rag relies on Albert parser
self.parser = DoclingServeParser()
def create_collection(self, name: str, description: Optional[str] = None) -> str:
"""
src/backend/chat/agent_rag/document_search/albert_api.py
+4 -30
View File
@@ -11,7 +11,7 @@ import requests
from chat.agent_rag.albert_api_constants import Searches
from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
from chat.agent_rag.document_converter.markitdown import DocumentConverter
from chat.agent_rag.document_converter.parser import DoclingServeParser
from chat.models import ChatConversation
logger = logging.getLogger(__name__)
@@ -105,32 +105,6 @@ class AlbertRagDocumentSearch:
document_page["content"] for document_page in response.json().get("data", [])
)
def parse_document(self, name: str, content_type: str, content: BytesIO):
"""
Parse the document and prepare it for the search operation.
This method should handle the logic to convert the document
into a format suitable for the Albert API.
Args:
name (str): The name of the document.
content_type (str): The MIME type of the document (e.g., "application/pdf").
content (BytesIO): The content of the document as a BytesIO stream.
Returns:
str: The document content in Markdown format.
"""
# Implement the parsing logic here
if content_type == "application/pdf":
# Handle PDF parsing
markdown_content = self._parse_pdf_document(
name=name, content_type=content_type, content=content
)
else:
markdown_content = DocumentConverter().convert_raw(
name=name, content_type=content_type, content=content
)
return markdown_content
def _store_document(self, name: str, content: str):
"""
@@ -156,16 +130,16 @@ class AlbertRagDocumentSearch:
logger.debug(response.json())
response.raise_for_status()
def parse_and_store_document(self, name: str, content_type: str, content: BytesIO):
def parse_and_store_document(self, name: str, content_type: str, content: bytes):
"""
Parse the document and store it in the Albert collection.
Args:
name (str): The name of the document.
content_type (str): The MIME type of the document (e.g., "application/pdf").
content (BytesIO): The content of the document as a BytesIO stream.
content (bytes): The content of the document as a BytesIO stream.
"""
document_content = self.parse_document(name, content_type, content)
document_content = DoclingServeParser().parse_document(name, content_type, content)
self._store_document(name, document_content)
return document_content
src/backend/conversations/settings.py
+12
View File
@@ -858,6 +858,18 @@ USER QUESTION:
environ_prefix=None,
)
# Docling
DOCLING_SERVE_URL = values.Value(
"http://docling-serve:5001",
environ_name="DOCLING_SERVE_URL",
environ_prefix = None,
)
DOCLING_SERVE_TIMEOUT = values.PositiveIntegerValue(
default=60, # seconds
environ_name="DOCLING_SERVE_TIMEOUT",
environ_prefix=None,
)
# Logging
# We want to make it easy to log to console but by default we log production
# to Sentry and don't want to log to console.