Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Quentin BEY
|
8434efd02d | ||
|
Berry den Hartog
|
174e1ca751 |
@@ -44,9 +44,6 @@ env.d/development/*
|
||||
!env.d/development/*.dist
|
||||
env.d/terraform
|
||||
|
||||
# Configuration
|
||||
**/conversations/configuration/llm/dev.json
|
||||
|
||||
# npm
|
||||
node_modules
|
||||
|
||||
@@ -82,6 +79,3 @@ db.sqlite3
|
||||
|
||||
# Docker compose override
|
||||
compose.override.yml
|
||||
|
||||
# Docling
|
||||
docling-models
|
||||
@@ -8,10 +8,6 @@ and this project adheres to
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- ✨(backend) add FindRagBackend
|
||||
|
||||
### Changed
|
||||
|
||||
- 📦️(front) update react
|
||||
|
||||
@@ -53,6 +53,9 @@ MAIL_YARN = $(COMPOSE_RUN) -w /app/src/mail node yarn
|
||||
# -- Frontend
|
||||
PATH_FRONT = ./src/frontend
|
||||
PATH_FRONT_CONVERSATIONS = $(PATH_FRONT)/apps/conversations
|
||||
FRONTEND_YARN = $(COMPOSE_RUN) -w /app/src/frontend node yarn
|
||||
FRONTEND_CONVERSATIONS_YARN = $(COMPOSE_RUN) -w /app/src/frontend/apps/conversations node yarn
|
||||
FRONTEND_CONVERSATIONS_YARN_3000 = $(COMPOSE_RUN) -p 3000:3000 -w /app/src/frontend/apps/conversations node yarn
|
||||
|
||||
# ==============================================================================
|
||||
# RULES
|
||||
@@ -337,20 +340,19 @@ help:
|
||||
|
||||
# Front
|
||||
frontend-development-install: ## install the frontend locally
|
||||
cd $(PATH_FRONT_CONVERSATIONS) && yarn
|
||||
@$(FRONTEND_CONVERSATIONS_YARN) install
|
||||
.PHONY: frontend-development-install
|
||||
|
||||
frontend-lint: ## run the frontend linter
|
||||
cd $(PATH_FRONT) && yarn lint
|
||||
@$(FRONTEND_YARN) lint
|
||||
.PHONY: frontend-lint
|
||||
|
||||
run-frontend-development: ## Run the frontend in development mode
|
||||
#@$(COMPOSE) stop frontend frontend-development
|
||||
cd $(PATH_FRONT_CONVERSATIONS) && yarn dev
|
||||
@$(FRONTEND_CONVERSATIONS_YARN_3000) dev
|
||||
.PHONY: run-frontend-development
|
||||
|
||||
frontend-i18n-extract: ## Extract the frontend translation inside a json to be used for crowdin
|
||||
cd $(PATH_FRONT) && yarn i18n:extract
|
||||
@$(FRONTEND_YARN) i18n:extract
|
||||
.PHONY: frontend-i18n-extract
|
||||
|
||||
frontend-i18n-generate: ## Generate the frontend json files used for crowdin
|
||||
@@ -360,7 +362,7 @@ frontend-i18n-generate: \
|
||||
.PHONY: frontend-i18n-generate
|
||||
|
||||
frontend-i18n-compile: ## Format the crowin json files used deploy to the apps
|
||||
cd $(PATH_FRONT) && yarn i18n:deploy
|
||||
@$(FRONTEND_YARN) i18n:deploy
|
||||
.PHONY: frontend-i18n-compile
|
||||
|
||||
# -- K8S
|
||||
@@ -374,10 +376,10 @@ start-tilt: ## start the kubernetes cluster using kind
|
||||
|
||||
bump-packages-version: VERSION_TYPE ?= minor
|
||||
bump-packages-version: ## bump the version of the project - VERSION_TYPE can be "major", "minor", "patch"
|
||||
cd ./src/mail && yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
cd ./src/frontend/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
cd ./src/frontend/apps/e2e/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
cd ./src/frontend/apps/conversations/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
cd ./src/frontend/packages/eslint-config-conversations/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
cd ./src/frontend/packages/i18n/ && yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
@$(COMPOSE_RUN) -w /app/src/mail node yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
@$(COMPOSE_RUN) -w /app/src/frontend node yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
@$(COMPOSE_RUN) -w /app/src/frontend/apps/e2e node yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
@$(COMPOSE_RUN) -w /app/src/frontend/apps/conversations node yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
@$(COMPOSE_RUN) -w /app/src/frontend/packages/eslint-config-conversations node yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
@$(COMPOSE_RUN) -w /app/src/frontend/packages/i18n node yarn version --no-git-tag-version --$(VERSION_TYPE)
|
||||
.PHONY: bump-packages-version
|
||||
|
||||
-11
@@ -71,9 +71,6 @@ services:
|
||||
- "host.docker.internal:host-gateway"
|
||||
ports:
|
||||
- "8071:8000"
|
||||
networks:
|
||||
- default
|
||||
- lasuite
|
||||
volumes:
|
||||
- ./src/backend:/app
|
||||
- ./data/static:/data/static
|
||||
@@ -92,9 +89,6 @@ services:
|
||||
image: nginx:1.25
|
||||
ports:
|
||||
- "8083:8083"
|
||||
networks:
|
||||
- default
|
||||
- lasuite
|
||||
volumes:
|
||||
- ./docker/files/etc/nginx/conf.d:/etc/nginx/conf.d:ro
|
||||
depends_on:
|
||||
@@ -183,8 +177,3 @@ services:
|
||||
kc_postgresql:
|
||||
condition: service_healthy
|
||||
restart: true
|
||||
|
||||
networks:
|
||||
lasuite:
|
||||
name: lasuite-network
|
||||
driver: bridge
|
||||
|
||||
@@ -95,9 +95,6 @@ These are the environment variables you can set for the `conversations-backend`
|
||||
| CACHES_KEY_PREFIX | The prefix used to every cache keys. | conversations |
|
||||
| THEME_CUSTOMIZATION_FILE_PATH | full path to the file customizing the theme. An example is provided in src/backend/conversations/configuration/theme/default.json | BASE_DIR/conversations/configuration/theme/default.json |
|
||||
| THEME_CUSTOMIZATION_CACHE_TIMEOUT | Cache duration for the customization settings | 86400 |
|
||||
| FIND_API_KEY | API key of Find | |
|
||||
| FIND_API_URL | URL of Find | `https://app-find/api` |
|
||||
| FIND_API_TIMEOUT | Find API timeout | 30 |
|
||||
|
||||
|
||||
## conversations-frontend image
|
||||
|
||||
@@ -244,9 +244,9 @@ For Mistral AI models using the Etalab platform:
|
||||
{
|
||||
"models": [
|
||||
{
|
||||
"hrid": "mistral-medium",
|
||||
"model_name": "mistral-medium-2508",
|
||||
"human_readable_name": "Mistral Medium (Etalab)",
|
||||
"hrid": "mistral-large",
|
||||
"model_name": "mistral-large-latest",
|
||||
"human_readable_name": "Mistral Large (Etalab)",
|
||||
"provider_name": "mistral-etalab",
|
||||
"profile": null,
|
||||
"settings": {
|
||||
|
||||
@@ -357,7 +357,6 @@ The RAG backend performs semantic search to find the most relevant content:
|
||||
rag_results = document_store.search(
|
||||
query,
|
||||
results_count=settings.BRAVE_RAG_WEB_SEARCH_CHUNK_NUMBER,
|
||||
**kwargs, # Additional search parameters like session with access_token
|
||||
)
|
||||
```
|
||||
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
"""Document parsers for RAG backends."""
|
||||
|
||||
import logging
|
||||
from io import BytesIO
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
import requests
|
||||
from docling.backend.pypdfium2_backend import PyPdfiumDocumentBackend
|
||||
from docling.datamodel.base_models import InputFormat
|
||||
from docling.datamodel.pipeline_options import PdfPipelineOptions, TableStructureOptions
|
||||
from docling.document_converter import DocumentConverter as DoclingDocumentConverter
|
||||
from docling.document_converter import PdfFormatOption
|
||||
from docling_core.types.io import DocumentStream
|
||||
|
||||
from chat.agent_rag.document_converter.markitdown import (
|
||||
DocumentConverter as MarkitdownDocumentConverter,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseParser:
|
||||
"""Base class for document parsers."""
|
||||
|
||||
def parse_document(self, name: str, content_type: str, content: BytesIO) -> str:
|
||||
"""
|
||||
Parse the document and prepare it for the search operation.
|
||||
This method should handle the logic to convert the document
|
||||
into a format suitable for storage.
|
||||
|
||||
Args:
|
||||
name (str): The name of the document.
|
||||
content_type (str): The MIME type of the document (e.g., "application/pdf").
|
||||
content (BytesIO): The content of the document as a BytesIO stream.
|
||||
|
||||
Returns:
|
||||
str: The document content in Markdown format.
|
||||
"""
|
||||
raise NotImplementedError("Must be implemented in subclass.")
|
||||
|
||||
|
||||
class AlbertParser(BaseParser):
|
||||
"""Document parser using Albert API for PDFs and DocumentConverter for other formats."""
|
||||
|
||||
endpoint = urljoin(settings.ALBERT_API_URL, "/v1/parse-beta")
|
||||
|
||||
def parse_pdf_document(self, name: str, content_type: str, content: bytes) -> str:
|
||||
"""Parse PDF document using Albert API."""
|
||||
response = requests.post(
|
||||
self.endpoint,
|
||||
headers={
|
||||
"Authorization": f"Bearer {settings.ALBERT_API_KEY}",
|
||||
},
|
||||
files={
|
||||
"file": (name, content, content_type),
|
||||
"output_format": (None, "markdown"),
|
||||
},
|
||||
timeout=settings.ALBERT_API_PARSE_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
return "\n\n".join(
|
||||
document_page["content"] for document_page in response.json().get("data", [])
|
||||
)
|
||||
|
||||
def parse_document(self, name: str, content_type: str, content: bytes) -> str:
|
||||
"""Parse document based on content type."""
|
||||
if content_type == "application/pdf":
|
||||
return self.parse_pdf_document(name=name, content_type=content_type, content=content)
|
||||
return MarkitdownDocumentConverter().convert_raw(
|
||||
name=name, content_type=content_type, content=content
|
||||
)
|
||||
|
||||
|
||||
class DoclingParser(BaseParser):
|
||||
"""Document parser using Docling's DocumentConverter."""
|
||||
|
||||
artifacts_path = "src/backend/docling-models"
|
||||
|
||||
def __init__(self):
|
||||
pipeline_options = PdfPipelineOptions(artifacts_path=self.artifacts_path)
|
||||
pipeline_options.do_ocr = True
|
||||
pipeline_options.do_table_structure = True
|
||||
pipeline_options.table_structure_options = TableStructureOptions(do_cell_matching=False)
|
||||
|
||||
self.converter = DoclingDocumentConverter(
|
||||
format_options={
|
||||
InputFormat.PDF: PdfFormatOption(
|
||||
pipeline_options=pipeline_options,
|
||||
backend=PyPdfiumDocumentBackend
|
||||
)}
|
||||
)
|
||||
|
||||
def parse_document(self, name: str, content_type: str, content: bytes) -> str:
|
||||
"""Parse document using Docling's DocumentConverter."""
|
||||
return self.converter.convert(
|
||||
DocumentStream(name=name, stream=BytesIO(content))
|
||||
).document.export_to_markdown()
|
||||
@@ -13,7 +13,7 @@ import requests
|
||||
|
||||
from chat.agent_rag.albert_api_constants import Searches
|
||||
from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
|
||||
from chat.agent_rag.document_converter.parser import DoclingParser
|
||||
from chat.agent_rag.document_converter.markitdown import DocumentConverter
|
||||
from chat.agent_rag.document_rag_backends.base_rag_backend import BaseRagBackend
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -26,6 +26,9 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
|
||||
|
||||
It provides methods to:
|
||||
- Create a collection for the search operation.
|
||||
- Parse documents and convert them to Markdown format:
|
||||
+ Handle PDF parsing using the Albert API.
|
||||
+ Use the DocumentConverter (markitdown) for other formats.
|
||||
- Store parsed documents in the Albert collection.
|
||||
- Perform a search operation using the Albert API.
|
||||
"""
|
||||
@@ -43,9 +46,10 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
|
||||
}
|
||||
self._collections_endpoint = urljoin(self._base_url, "/v1/collections")
|
||||
self._documents_endpoint = urljoin(self._base_url, "/v1/documents")
|
||||
self._pdf_parser_endpoint = urljoin(self._base_url, "/v1/parse-beta")
|
||||
self._search_endpoint = urljoin(self._base_url, "/v1/search")
|
||||
|
||||
self._default_collection_description = "Temporary collection for RAG document search"
|
||||
self.parser = DoclingParser()
|
||||
|
||||
def create_collection(self, name: str, description: Optional[str] = None) -> str:
|
||||
"""
|
||||
@@ -110,7 +114,59 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
def store_document(self, name: str, content: str, **kwargs) -> None:
|
||||
def parse_pdf_document(self, name: str, content_type: str, content: BytesIO) -> str:
|
||||
"""
|
||||
Parse the PDF document content and return the text content.
|
||||
This method should handle the logic to convert the PDF into
|
||||
a format suitable for the Albert API.
|
||||
"""
|
||||
response = requests.post(
|
||||
self._pdf_parser_endpoint,
|
||||
headers=self._headers,
|
||||
files={
|
||||
"file": (
|
||||
name,
|
||||
content,
|
||||
content_type,
|
||||
), # Use the name as the filename in the request
|
||||
"output_format": (None, "markdown"), # Specify the output format as Markdown,
|
||||
},
|
||||
timeout=settings.ALBERT_API_PARSE_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
return "\n\n".join(
|
||||
document_page["content"] for document_page in response.json().get("data", [])
|
||||
)
|
||||
|
||||
def parse_document(self, name: str, content_type: str, content: BytesIO):
|
||||
"""
|
||||
Parse the document and prepare it for the search operation.
|
||||
This method should handle the logic to convert the document
|
||||
into a format suitable for the Albert API.
|
||||
|
||||
Args:
|
||||
name (str): The name of the document.
|
||||
content_type (str): The MIME type of the document (e.g., "application/pdf").
|
||||
content (BytesIO): The content of the document as a BytesIO stream.
|
||||
|
||||
Returns:
|
||||
str: The document content in Markdown format.
|
||||
"""
|
||||
# Implement the parsing logic here
|
||||
if content_type == "application/pdf":
|
||||
# Handle PDF parsing
|
||||
markdown_content = self.parse_pdf_document(
|
||||
name=name, content_type=content_type, content=content
|
||||
)
|
||||
else:
|
||||
markdown_content = DocumentConverter().convert_raw(
|
||||
name=name, content_type=content_type, content=content
|
||||
)
|
||||
|
||||
return markdown_content
|
||||
|
||||
def store_document(self, name: str, content: str) -> None:
|
||||
"""
|
||||
Store the document content in the Albert collection.
|
||||
This method should handle the logic to send the document content to the Albert API.
|
||||
@@ -118,7 +174,6 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
|
||||
Args:
|
||||
name (str): The name of the document.
|
||||
content (str): The content of the document in Markdown format.
|
||||
**kwargs: Additional arguments.
|
||||
"""
|
||||
response = requests.post(
|
||||
urljoin(self._base_url, self._documents_endpoint),
|
||||
@@ -133,7 +188,7 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
|
||||
logger.debug(response.json())
|
||||
response.raise_for_status()
|
||||
|
||||
async def astore_document(self, name: str, content: str, **kwargs) -> None:
|
||||
async def astore_document(self, name: str, content: str) -> None:
|
||||
"""
|
||||
Store the document content in the Albert collection.
|
||||
This method should handle the logic to send the document content to the Albert API.
|
||||
@@ -141,7 +196,6 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
|
||||
Args:
|
||||
name (str): The name of the document.
|
||||
content (str): The content of the document in Markdown format.
|
||||
**kwargs: Additional arguments.
|
||||
"""
|
||||
async with httpx.AsyncClient(timeout=settings.ALBERT_API_TIMEOUT) as client:
|
||||
response = await client.post(
|
||||
@@ -159,14 +213,13 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
|
||||
logger.debug(response.json())
|
||||
response.raise_for_status()
|
||||
|
||||
def search(self, query: str, results_count: int = 4, **kwargs) -> RAGWebResults:
|
||||
def search(self, query, results_count: int = 4) -> RAGWebResults:
|
||||
"""
|
||||
Perform a search using the Albert API based on the provided query.
|
||||
|
||||
Args:
|
||||
query (str): The search query.
|
||||
results_count (int): The number of results to return.
|
||||
**kwargs: Additional arguments.
|
||||
|
||||
Returns:
|
||||
RAGWebResults: The search results.
|
||||
@@ -203,14 +256,13 @@ class AlbertRagBackend(BaseRagBackend): # pylint: disable=too-many-instance-att
|
||||
),
|
||||
)
|
||||
|
||||
async def asearch(self, query, results_count: int = 4, **kwargs) -> RAGWebResults:
|
||||
async def asearch(self, query, results_count: int = 4) -> RAGWebResults:
|
||||
"""
|
||||
Perform an asynchronous search using the Albert API based on the provided query.
|
||||
|
||||
Args:
|
||||
query (str): The search query.
|
||||
results_count (int): The number of results to return.
|
||||
**kwargs: Additional arguments.
|
||||
|
||||
Returns:
|
||||
RAGWebResults: The search results.
|
||||
|
||||
@@ -8,7 +8,6 @@ from typing import List, Optional
|
||||
from asgiref.sync import sync_to_async
|
||||
|
||||
from chat.agent_rag.constants import RAGWebResults
|
||||
from chat.agent_rag.document_converter.parser import BaseParser
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -39,7 +38,6 @@ class BaseRagBackend:
|
||||
self.collection_id = collection_id
|
||||
self.read_only_collection_id = read_only_collection_id or []
|
||||
self._default_collection_description = "Temporary collection for RAG document search"
|
||||
self.parser: BaseParser = BaseParser()
|
||||
|
||||
def get_all_collection_ids(self) -> List[str]:
|
||||
"""
|
||||
@@ -55,7 +53,7 @@ class BaseRagBackend:
|
||||
|
||||
collection_ids = []
|
||||
if self.collection_id:
|
||||
collection_ids.append(self.collection_id)
|
||||
collection_ids.append(int(self.collection_id))
|
||||
if self.read_only_collection_id:
|
||||
collection_ids.extend(
|
||||
[int(collection_id) for collection_id in self.read_only_collection_id]
|
||||
@@ -90,9 +88,9 @@ class BaseRagBackend:
|
||||
Returns:
|
||||
str: The document content in Markdown format.
|
||||
"""
|
||||
return self.parser.parse_document(name, content_type, content)
|
||||
raise NotImplementedError("Must be implemented in subclass.")
|
||||
|
||||
def store_document(self, name: str, content: str, **kwargs) -> None:
|
||||
def store_document(self, name: str, content: str) -> None:
|
||||
"""
|
||||
Store the document content in the collection.
|
||||
This method should handle the logic to send the document content to the API.
|
||||
@@ -100,11 +98,10 @@ class BaseRagBackend:
|
||||
Args:
|
||||
name (str): The name of the document.
|
||||
content (str): The content of the document in Markdown format.
|
||||
**kwargs: Additional arguments. ex: "user_sub" for access control.
|
||||
"""
|
||||
raise NotImplementedError("Must be implemented in subclass.")
|
||||
|
||||
async def astore_document(self, name: str, content: str, **kwargs) -> None:
|
||||
async def astore_document(self, name: str, content: str) -> None:
|
||||
"""
|
||||
Store the document content in the collection.
|
||||
This method should handle the logic to send the document content to the API.
|
||||
@@ -112,13 +109,10 @@ class BaseRagBackend:
|
||||
Args:
|
||||
name (str): The name of the document.
|
||||
content (str): The content of the document in Markdown format.
|
||||
**kwargs: Additional arguments. ex: "user_sub" for access control.
|
||||
"""
|
||||
return await sync_to_async(self.store_document)(name=name, content=content, **kwargs)
|
||||
return await sync_to_async(self.store_document)(name=name, content=content)
|
||||
|
||||
def parse_and_store_document(
|
||||
self, name: str, content_type: str, content: BytesIO, **kwargs
|
||||
) -> str:
|
||||
def parse_and_store_document(self, name: str, content_type: str, content: BytesIO) -> str:
|
||||
"""
|
||||
Parse the document and store it in the Albert collection.
|
||||
|
||||
@@ -126,13 +120,12 @@ class BaseRagBackend:
|
||||
name (str): The name of the document.
|
||||
content_type (str): The MIME type of the document (e.g., "application/pdf").
|
||||
content (BytesIO): The content of the document as a BytesIO stream.
|
||||
**kwargs: Additional arguments. ex: "user_sub" for access control.
|
||||
"""
|
||||
if not self.collection_id:
|
||||
raise RuntimeError("The RAG backend requires collection_id")
|
||||
|
||||
document_content = self.parse_document(name, content_type, content)
|
||||
self.store_document(name, document_content, **kwargs)
|
||||
self.store_document(name, document_content)
|
||||
return document_content
|
||||
|
||||
def delete_collection(self) -> None:
|
||||
@@ -149,27 +142,17 @@ class BaseRagBackend:
|
||||
"""
|
||||
return await sync_to_async(self.delete_collection)()
|
||||
|
||||
def search(self, query: str, results_count: int = 4, **kwargs) -> RAGWebResults:
|
||||
def search(self, query, results_count: int = 4) -> RAGWebResults:
|
||||
"""
|
||||
Search the collection for the given query.
|
||||
|
||||
Args:
|
||||
query: The search query string.
|
||||
results_count: Number of results to return.
|
||||
**kwargs: Additional arguments. ex: 'session' for OIDC authentication.
|
||||
"""
|
||||
raise NotImplementedError("Must be implemented in subclass.")
|
||||
|
||||
async def asearch(self, query: str, results_count: int = 4, **kwargs) -> RAGWebResults:
|
||||
async def asearch(self, query, results_count: int = 4) -> RAGWebResults:
|
||||
"""
|
||||
Search the collection for the given query asynchronously.
|
||||
|
||||
Args:
|
||||
query: The search query string.
|
||||
results_count: Number of results to return.
|
||||
**kwargs: Additional arguments. ex: 'session' for OIDC authentication.
|
||||
Search the collection for the given query.
|
||||
"""
|
||||
return await sync_to_async(self.search)(query=query, results_count=results_count, **kwargs)
|
||||
return await sync_to_async(self.search)(query=query, results_count=results_count)
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
"""Implementation of the Find API for RAG document search."""
|
||||
|
||||
import logging
|
||||
import uuid
|
||||
from typing import List, Optional
|
||||
from urllib.parse import urljoin
|
||||
from uuid import uuid4
|
||||
|
||||
from django.conf import settings
|
||||
from django.utils import timezone
|
||||
|
||||
import requests
|
||||
|
||||
from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
|
||||
from chat.agent_rag.document_converter.parser import DoclingParser
|
||||
from chat.agent_rag.document_rag_backends.base_rag_backend import BaseRagBackend
|
||||
from utils.oidc import with_fresh_access_token
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
SUPPORTED_LANGUAGE_CODES = ["en", "fr", "de", "nl"]
|
||||
|
||||
|
||||
class FindRagBackend(BaseRagBackend):
|
||||
"""
|
||||
This class is a placeholder for the Find API implementation.
|
||||
It is designed to be used with the RAG (Retrieval-Augmented Generation) document search system.
|
||||
|
||||
It provides methods to:
|
||||
- Store parsed documents in the Find index.
|
||||
- Perform a search operation using the Find API.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
collection_id: Optional[str] = None,
|
||||
read_only_collection_id: Optional[List[str]] = None,
|
||||
):
|
||||
# Initialize any necessary parameters or configurations here
|
||||
super().__init__(collection_id, read_only_collection_id)
|
||||
self.api_key = settings.FIND_API_KEY
|
||||
self.search_endpoint = "api/v1.0/documents/search/"
|
||||
self.indexing_endpoint = "api/v1.0/documents/index/"
|
||||
self.parser = DoclingParser()
|
||||
|
||||
def create_collection(self, name: str, description: Optional[str] = None) -> str:
|
||||
"""
|
||||
init collection_id
|
||||
"""
|
||||
self.collection_id = self.collection_id or str(uuid.uuid4())
|
||||
return self.collection_id
|
||||
|
||||
def delete_collection(self) -> None:
|
||||
"""
|
||||
Deletion not available
|
||||
"""
|
||||
logger.warning("deletion of collections is not yet supported in FindRagBackend")
|
||||
|
||||
def store_document(self, name: str, content: str, **kwargs) -> None:
|
||||
"""
|
||||
index document in Find
|
||||
|
||||
Args:
|
||||
name (str): The name of the document.
|
||||
content (str): The content of the document in Markdown format.
|
||||
user_sub (str): The user subject identifier for access control.
|
||||
"""
|
||||
logger.debug("index document '%s' in Find", name)
|
||||
|
||||
user_sub = kwargs.get("user_sub")
|
||||
if not user_sub:
|
||||
raise ValueError("user_sub is required to store document in FindRagBackend")
|
||||
|
||||
response = requests.post(
|
||||
urljoin(settings.FIND_API_URL, self.indexing_endpoint),
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={
|
||||
"id": str(uuid4()),
|
||||
"title": str(name) or "",
|
||||
"depth": 0,
|
||||
"path": str(name) or "",
|
||||
"numchild": 0,
|
||||
"content": content or "",
|
||||
"created_at": timezone.now().isoformat(),
|
||||
"updated_at": timezone.now().isoformat(),
|
||||
"tags": [f"collection-{self.collection_id}"],
|
||||
"size": len(content.encode("utf-8")),
|
||||
"users": [user_sub],
|
||||
"groups": [],
|
||||
"reach": "authenticated",
|
||||
"is_active": True,
|
||||
},
|
||||
timeout=settings.FIND_API_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
@with_fresh_access_token
|
||||
def search(self, query: str, results_count: int = 4, **kwargs) -> RAGWebResults:
|
||||
"""
|
||||
Perform a search using the Find API.
|
||||
Uses the user's OIDC token from the request session.
|
||||
|
||||
Args:
|
||||
query: The search query.
|
||||
results_count: Number of results to return.
|
||||
**kwargs: Additional arguments. Expected: 'session' containing OIDC tokens,
|
||||
|
||||
Returns:
|
||||
RAGWebResults: The search results.
|
||||
"""
|
||||
logger.debug("search documents in Find with query '%s'", query)
|
||||
|
||||
response = requests.post(
|
||||
urljoin(settings.FIND_API_URL, self.search_endpoint),
|
||||
headers={"Authorization": f"Bearer {kwargs['session'].get('oidc_access_token')}"},
|
||||
json={
|
||||
"q": query,
|
||||
"tags": [
|
||||
f"collection-{collection_id}" for collection_id in self.get_all_collection_ids()
|
||||
],
|
||||
"k": results_count,
|
||||
},
|
||||
timeout=settings.FIND_API_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
return RAGWebResults(
|
||||
data=[
|
||||
RAGWebResult(
|
||||
url=get_language_value(result["_source"], "title"),
|
||||
content=get_language_value(result["_source"], "content"),
|
||||
score=result["_score"],
|
||||
)
|
||||
for result in response.json()
|
||||
],
|
||||
usage=RAGWebUsage(
|
||||
prompt_tokens=0,
|
||||
completion_tokens=0,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def get_language_value(source, language_field):
|
||||
"""
|
||||
extract the value of the language field with the correct language_code extension.
|
||||
"title" and "content" have extensions like "title.en" or "title.fr".
|
||||
get_language_value will return the value regardless of the extension.
|
||||
"""
|
||||
for language_code in SUPPORTED_LANGUAGE_CODES:
|
||||
if f"{language_field}.{language_code}" in source:
|
||||
return source[f"{language_field}.{language_code}"]
|
||||
raise ValueError(f"No '{language_field}' field with any supported language code in object")
|
||||
@@ -11,7 +11,7 @@ import requests
|
||||
|
||||
from chat.agent_rag.albert_api_constants import Searches
|
||||
from chat.agent_rag.constants import RAGWebResult, RAGWebResults, RAGWebUsage
|
||||
from chat.agent_rag.document_converter.parser import DoclingParser
|
||||
from chat.agent_rag.document_converter.markitdown import DocumentConverter
|
||||
from chat.models import ChatConversation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -80,6 +80,58 @@ class AlbertRagDocumentSearch:
|
||||
self.conversation.collection_id = str(response.json()["id"])
|
||||
return True
|
||||
|
||||
def _parse_pdf_document(self, name: str, content_type: str, content: BytesIO) -> str:
|
||||
"""
|
||||
Parse the PDF document content and return the text content.
|
||||
This method should handle the logic to convert the PDF into
|
||||
a format suitable for the Albert API.
|
||||
"""
|
||||
response = requests.post(
|
||||
self._pdf_parser_endpoint,
|
||||
headers=self._headers,
|
||||
files={
|
||||
"file": (
|
||||
name,
|
||||
content,
|
||||
content_type,
|
||||
), # Use the name as the filename in the request
|
||||
"output_format": (None, "markdown"), # Specify the output format as Markdown,
|
||||
},
|
||||
timeout=settings.ALBERT_API_PARSE_TIMEOUT,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
return "\n\n".join(
|
||||
document_page["content"] for document_page in response.json().get("data", [])
|
||||
)
|
||||
|
||||
def parse_document(self, name: str, content_type: str, content: BytesIO):
|
||||
"""
|
||||
Parse the document and prepare it for the search operation.
|
||||
This method should handle the logic to convert the document
|
||||
into a format suitable for the Albert API.
|
||||
|
||||
Args:
|
||||
name (str): The name of the document.
|
||||
content_type (str): The MIME type of the document (e.g., "application/pdf").
|
||||
content (BytesIO): The content of the document as a BytesIO stream.
|
||||
|
||||
Returns:
|
||||
str: The document content in Markdown format.
|
||||
"""
|
||||
# Implement the parsing logic here
|
||||
if content_type == "application/pdf":
|
||||
# Handle PDF parsing
|
||||
markdown_content = self._parse_pdf_document(
|
||||
name=name, content_type=content_type, content=content
|
||||
)
|
||||
else:
|
||||
markdown_content = DocumentConverter().convert_raw(
|
||||
name=name, content_type=content_type, content=content
|
||||
)
|
||||
|
||||
return markdown_content
|
||||
|
||||
def _store_document(self, name: str, content: str):
|
||||
"""
|
||||
Store the document content in the Albert collection.
|
||||
@@ -104,7 +156,7 @@ class AlbertRagDocumentSearch:
|
||||
logger.debug(response.json())
|
||||
response.raise_for_status()
|
||||
|
||||
def parse_and_store_document(self, name: str, content_type: str, content: bytes):
|
||||
def parse_and_store_document(self, name: str, content_type: str, content: BytesIO):
|
||||
"""
|
||||
Parse the document and store it in the Albert collection.
|
||||
|
||||
@@ -113,9 +165,7 @@ class AlbertRagDocumentSearch:
|
||||
content_type (str): The MIME type of the document (e.g., "application/pdf").
|
||||
content (BytesIO): The content of the document as a BytesIO stream.
|
||||
"""
|
||||
document_content = DoclingParser().parse_document(
|
||||
name=name, content_type=content_type, content=content
|
||||
)
|
||||
document_content = self.parse_document(name, content_type, content)
|
||||
self._store_document(name, document_content)
|
||||
return document_content
|
||||
|
||||
|
||||
@@ -92,7 +92,6 @@ class ContextDeps:
|
||||
|
||||
conversation: models.ChatConversation
|
||||
user: User
|
||||
session: Optional[Dict] = None
|
||||
web_search_enabled: bool = False
|
||||
|
||||
|
||||
@@ -107,14 +106,7 @@ def get_model_configuration(model_hrid: str):
|
||||
class AIAgentService: # pylint: disable=too-many-instance-attributes
|
||||
"""Service class for AI-related operations (Pydantic-AI edition)."""
|
||||
|
||||
def __init__( # pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
self,
|
||||
conversation: models.ChatConversation,
|
||||
user,
|
||||
session=None,
|
||||
model_hrid=None,
|
||||
language=None,
|
||||
):
|
||||
def __init__(self, conversation: models.ChatConversation, user, model_hrid=None, language=None):
|
||||
"""
|
||||
Initialize the AI agent service.
|
||||
|
||||
@@ -144,7 +136,6 @@ class AIAgentService: # pylint: disable=too-many-instance-attributes
|
||||
self._context_deps = ContextDeps(
|
||||
conversation=conversation,
|
||||
user=user,
|
||||
session=session,
|
||||
web_search_enabled=self._is_web_search_enabled,
|
||||
)
|
||||
|
||||
@@ -287,7 +278,6 @@ class AIAgentService: # pylint: disable=too-many-instance-attributes
|
||||
name=document.identifier,
|
||||
content_type=document.media_type,
|
||||
content=document_data,
|
||||
user_sub=self.user.sub,
|
||||
)
|
||||
else:
|
||||
# Remote URL
|
||||
@@ -297,7 +287,6 @@ class AIAgentService: # pylint: disable=too-many-instance-attributes
|
||||
name=document.identifier,
|
||||
content_type=document.media_type,
|
||||
content=document.data,
|
||||
user_sub=self.user.sub,
|
||||
)
|
||||
|
||||
if not document.media_type.startswith("text/"):
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
"""
|
||||
Unit tests for the DocumentConverter.
|
||||
|
||||
Only for coverage as the DocumentConverter is a simple wrapper around MarkItDown.
|
||||
"""
|
||||
|
||||
from io import BytesIO
|
||||
|
||||
from docling.document_converter import DocumentConverter
|
||||
from docling_core.types.io import DocumentStream
|
||||
|
||||
|
||||
def main():
|
||||
"""Test that the DocumentConverter calls the underlying MarkItDown converter."""
|
||||
file_path = "test.pdf"
|
||||
converter = DocumentConverter()
|
||||
|
||||
# Convert from file content instead of file path
|
||||
with open(file_path, "rb") as file:
|
||||
content = file.read()
|
||||
stream = DocumentStream(name="test.pdf", stream=BytesIO(content))
|
||||
result = converter.convert(stream)
|
||||
markdown = result.document.export_to_markdown()
|
||||
|
||||
assert markdown == "Document PDF test"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,90 +0,0 @@
|
||||
%PDF-1.4
|
||||
%Çì�¢
|
||||
5 0 obj
|
||||
<</Length 6 0 R/Filter /FlateDecode>>
|
||||
stream
|
||||
xœMŽ±NÄ0 †÷<…Çd¨ÏNœ¸^OÀÀ§l'¦Šc*¨âx’RªÚƒÿóo{BŽ@=ÿ›iv nq#¦ «xì§ÎÕ�.
|
||||
ÍQoŽÐÌ„WÆ „#h!¨³»ú‡À˜5Sò_a Œ&¦â§�°•Ÿƒ4‡!¢ÊÅ¿ÿÑϽwÊ%çÑC—Y4[ò/a�ö³n‡D¢
|
||||
‹æhû¨Z<nØö‡�1F3Ýaj–·úì «{mù µi:uendstream
|
||||
endobj
|
||||
6 0 obj
|
||||
180
|
||||
endobj
|
||||
4 0 obj
|
||||
<</Type/Page/MediaBox [0 0 595 842]
|
||||
/Rotate 0/Parent 3 0 R
|
||||
/Resources<</ProcSet[/PDF /Text]
|
||||
/Font 8 0 R
|
||||
>>
|
||||
/Contents 5 0 R
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<< /Type /Pages /Kids [
|
||||
4 0 R
|
||||
] /Count 1
|
||||
>>
|
||||
endobj
|
||||
1 0 obj
|
||||
<</Type /Catalog /Pages 3 0 R
|
||||
/Metadata 9 0 R
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<</R7
|
||||
7 0 R>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<</BaseFont/Times-Roman/Type/Font
|
||||
/Subtype/Type1>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<</Type/Metadata
|
||||
/Subtype/XML/Length 1549>>stream
|
||||
<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
|
||||
<?adobe-xap-filters esc="CRLF"?>
|
||||
<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
|
||||
<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
|
||||
<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:pdf='http://ns.adobe.com/pdf/1.3/'><pdf:Producer>GPL Ghostscript 9.06</pdf:Producer>
|
||||
<pdf:Keywords>()</pdf:Keywords>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:xmp='http://ns.adobe.com/xap/1.0/'><xmp:ModifyDate>2014-12-22T00:49:20+01:00</xmp:ModifyDate>
|
||||
<xmp:CreateDate>2014-12-22T00:49:20+01:00</xmp:CreateDate>
|
||||
<xmp:CreatorTool>PDFCreator Version 1.6.0</xmp:CreatorTool></rdf:Description>
|
||||
<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c'/>
|
||||
<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>test_word</rdf:li></rdf:Alt></dc:title><dc:creator><rdf:Seq><rdf:li>Seb</rdf:li></rdf:Seq></dc:creator><dc:description><rdf:Seq><rdf:li>()</rdf:li></rdf:Seq></dc:description></rdf:Description>
|
||||
</rdf:RDF>
|
||||
</x:xmpmeta>
|
||||
|
||||
|
||||
<?xpacket end='w'?>
|
||||
endstream
|
||||
endobj
|
||||
2 0 obj
|
||||
<</Producer(GPL Ghostscript 9.06)
|
||||
/CreationDate(D:20141222004920+01'00')
|
||||
/ModDate(D:20141222004920+01'00')
|
||||
/Title(\376\377\000t\000e\000s\000t\000_\000w\000o\000r\000d)
|
||||
/Creator(\376\377\000P\000D\000F\000C\000r\000e\000a\000t\000o\000r\000 \000V\000e\000r\000s\000i\000o\000n\000 \0001\000.\0006\000.\0000)
|
||||
/Author(\376\377\000S\000e\000b)
|
||||
/Keywords()
|
||||
/Subject()>>endobj
|
||||
xref
|
||||
0 10
|
||||
0000000000 65535 f
|
||||
0000000484 00000 n
|
||||
0000002268 00000 n
|
||||
0000000425 00000 n
|
||||
0000000284 00000 n
|
||||
0000000015 00000 n
|
||||
0000000265 00000 n
|
||||
0000000577 00000 n
|
||||
0000000548 00000 n
|
||||
0000000643 00000 n
|
||||
trailer
|
||||
<< /Size 10 /Root 1 0 R /Info 2 0 R
|
||||
/ID [<0CB231047435B33BCE0B1C6881DCF011><0CB231047435B33BCE0B1C6881DCF011>]
|
||||
>>
|
||||
startxref
|
||||
2648
|
||||
%%EOF
|
||||
@@ -1,18 +0,0 @@
|
||||
"""
|
||||
Unit tests for the DoclingParser.
|
||||
"""
|
||||
from chat.agent_rag.document_converter.parser import DoclingParser
|
||||
|
||||
|
||||
def test_document_converter():
|
||||
"""Test that the DocumentConverter calls the underlying MarkItDown converter."""
|
||||
file_name = "test"
|
||||
content_type = "application/pdf"
|
||||
file_path = "src/backend/chat/tests/data/test.pdf"
|
||||
parser = DoclingParser()
|
||||
|
||||
with open(file_path, "rb") as file:
|
||||
content = file.read()
|
||||
result = parser.parse_document(name= file_name, content_type= content_type, content= content)
|
||||
|
||||
assert "Document PDF test" in result
|
||||
@@ -5,21 +5,28 @@ Only for coverage as the DocumentConverter is a simple wrapper around MarkItDown
|
||||
"""
|
||||
|
||||
from io import BytesIO
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from chat.agent_rag.document_converter.markitdown import DocumentConverter
|
||||
|
||||
|
||||
def test_document_converter():
|
||||
@patch("chat.agent_rag.document_converter.markitdown.MarkItDown")
|
||||
def test_document_converter(mock_markitdown: MagicMock):
|
||||
"""Test that the DocumentConverter calls the underlying MarkItDown converter."""
|
||||
file_path = "src/backend/chat/tests/data/test.pdf"
|
||||
mock_conversion = MagicMock()
|
||||
mock_conversion.text_content = "converted text"
|
||||
mock_markitdown.return_value.convert_stream.return_value = mock_conversion
|
||||
|
||||
converter = DocumentConverter()
|
||||
|
||||
with open(file_path, "rb") as file:
|
||||
content = file.read()
|
||||
result = converter.convert_raw(
|
||||
name="test.pdf",
|
||||
content_type="application/pdf",
|
||||
content=content,
|
||||
)
|
||||
result = converter.convert_raw(
|
||||
name="test.pdf",
|
||||
content_type="application/pdf",
|
||||
content=b"test content",
|
||||
)
|
||||
|
||||
assert result == "Document PDF test\n\n"
|
||||
assert result == "converted text"
|
||||
converter.converter.convert_stream.assert_called_once() # pylint: disable=no-member
|
||||
args, kwargs = converter.converter.convert_stream.call_args # pylint: disable=no-member
|
||||
assert isinstance(args[0], BytesIO)
|
||||
assert kwargs["file_extension"] == ".pdf"
|
||||
|
||||
@@ -1,90 +0,0 @@
|
||||
%PDF-1.4
|
||||
%Çì�¢
|
||||
5 0 obj
|
||||
<</Length 6 0 R/Filter /FlateDecode>>
|
||||
stream
|
||||
xœMŽ±NÄ0 †÷<…Çd¨ÏNœ¸^OÀÀ§l'¦Šc*¨âx’RªÚƒÿóo{BŽ@=ÿ›iv nq#¦ «xì§ÎÕ�.
|
||||
ÍQoŽÐÌ„WÆ „#h!¨³»ú‡À˜5Sò_a Œ&¦â§�°•Ÿƒ4‡!¢ÊÅ¿ÿÑϽwÊ%çÑC—Y4[ò/a�ö³n‡D¢
|
||||
‹æhû¨Z<nØö‡�1F3Ýaj–·úì «{mù µi:uendstream
|
||||
endobj
|
||||
6 0 obj
|
||||
180
|
||||
endobj
|
||||
4 0 obj
|
||||
<</Type/Page/MediaBox [0 0 595 842]
|
||||
/Rotate 0/Parent 3 0 R
|
||||
/Resources<</ProcSet[/PDF /Text]
|
||||
/Font 8 0 R
|
||||
>>
|
||||
/Contents 5 0 R
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<< /Type /Pages /Kids [
|
||||
4 0 R
|
||||
] /Count 1
|
||||
>>
|
||||
endobj
|
||||
1 0 obj
|
||||
<</Type /Catalog /Pages 3 0 R
|
||||
/Metadata 9 0 R
|
||||
>>
|
||||
endobj
|
||||
8 0 obj
|
||||
<</R7
|
||||
7 0 R>>
|
||||
endobj
|
||||
7 0 obj
|
||||
<</BaseFont/Times-Roman/Type/Font
|
||||
/Subtype/Type1>>
|
||||
endobj
|
||||
9 0 obj
|
||||
<</Type/Metadata
|
||||
/Subtype/XML/Length 1549>>stream
|
||||
<?xpacket begin='' id='W5M0MpCehiHzreSzNTczkc9d'?>
|
||||
<?adobe-xap-filters esc="CRLF"?>
|
||||
<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 2.9.1-13, framework 1.6'>
|
||||
<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
|
||||
<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:pdf='http://ns.adobe.com/pdf/1.3/'><pdf:Producer>GPL Ghostscript 9.06</pdf:Producer>
|
||||
<pdf:Keywords>()</pdf:Keywords>
|
||||
</rdf:Description>
|
||||
<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:xmp='http://ns.adobe.com/xap/1.0/'><xmp:ModifyDate>2014-12-22T00:49:20+01:00</xmp:ModifyDate>
|
||||
<xmp:CreateDate>2014-12-22T00:49:20+01:00</xmp:CreateDate>
|
||||
<xmp:CreatorTool>PDFCreator Version 1.6.0</xmp:CreatorTool></rdf:Description>
|
||||
<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:xapMM='http://ns.adobe.com/xap/1.0/mm/' xapMM:DocumentID='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c'/>
|
||||
<rdf:Description rdf:about='uuid:81d69fb9-8bc7-11e4-0000-66b1dd18110c' xmlns:dc='http://purl.org/dc/elements/1.1/' dc:format='application/pdf'><dc:title><rdf:Alt><rdf:li xml:lang='x-default'>test_word</rdf:li></rdf:Alt></dc:title><dc:creator><rdf:Seq><rdf:li>Seb</rdf:li></rdf:Seq></dc:creator><dc:description><rdf:Seq><rdf:li>()</rdf:li></rdf:Seq></dc:description></rdf:Description>
|
||||
</rdf:RDF>
|
||||
</x:xmpmeta>
|
||||
|
||||
|
||||
<?xpacket end='w'?>
|
||||
endstream
|
||||
endobj
|
||||
2 0 obj
|
||||
<</Producer(GPL Ghostscript 9.06)
|
||||
/CreationDate(D:20141222004920+01'00')
|
||||
/ModDate(D:20141222004920+01'00')
|
||||
/Title(\376\377\000t\000e\000s\000t\000_\000w\000o\000r\000d)
|
||||
/Creator(\376\377\000P\000D\000F\000C\000r\000e\000a\000t\000o\000r\000 \000V\000e\000r\000s\000i\000o\000n\000 \0001\000.\0006\000.\0000)
|
||||
/Author(\376\377\000S\000e\000b)
|
||||
/Keywords()
|
||||
/Subject()>>endobj
|
||||
xref
|
||||
0 10
|
||||
0000000000 65535 f
|
||||
0000000484 00000 n
|
||||
0000002268 00000 n
|
||||
0000000425 00000 n
|
||||
0000000284 00000 n
|
||||
0000000015 00000 n
|
||||
0000000265 00000 n
|
||||
0000000577 00000 n
|
||||
0000000548 00000 n
|
||||
0000000643 00000 n
|
||||
trailer
|
||||
<< /Size 10 /Root 1 0 R /Info 2 0 R
|
||||
/ID [<0CB231047435B33BCE0B1C6881DCF011><0CB231047435B33BCE0B1C6881DCF011>]
|
||||
>>
|
||||
startxref
|
||||
2648
|
||||
%%EOF
|
||||
@@ -38,6 +38,9 @@ def brave_settings(settings):
|
||||
settings.BRAVE_SEARCH_EXTRA_SNIPPETS = True
|
||||
settings.BRAVE_SUMMARIZATION_ENABLED = False
|
||||
settings.BRAVE_CACHE_TTL = 3600
|
||||
settings.RAG_DOCUMENT_SEARCH_BACKEND = (
|
||||
"chat.agent_rag.document_rag_backends.albert_rag_backend.AlbertRagBackend"
|
||||
)
|
||||
settings.BRAVE_RAG_WEB_SEARCH_CHUNK_NUMBER = 5
|
||||
|
||||
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
"""Common test fixtures for chat views tests."""
|
||||
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_process_request():
|
||||
"""
|
||||
Mock process_request to bypass OIDC authentication in tests.
|
||||
"""
|
||||
with mock.patch(
|
||||
"lasuite.oidc_login.decorators.RefreshOIDCAccessToken.process_request"
|
||||
) as mocked_process_request:
|
||||
mocked_process_request.return_value = None
|
||||
yield mocked_process_request
|
||||
+22
-74
@@ -8,7 +8,6 @@ import logging
|
||||
from io import BytesIO
|
||||
from unittest import mock
|
||||
|
||||
from django.contrib.sessions.backends.cache import SessionStore
|
||||
from django.utils import formats, timezone
|
||||
|
||||
import httpx
|
||||
@@ -42,49 +41,28 @@ from chat.tests.utils import replace_uuids_with_placeholder
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
autouse=True,
|
||||
params=[
|
||||
"chat.agent_rag.document_rag_backends.find_rag_backend.FindRagBackend",
|
||||
"chat.agent_rag.document_rag_backends.albert_rag_backend.AlbertRagBackend",
|
||||
],
|
||||
)
|
||||
def ai_settings(request, settings):
|
||||
@pytest.fixture(autouse=True)
|
||||
def ai_settings(settings):
|
||||
"""Fixture to set AI service URLs for testing."""
|
||||
|
||||
# enable on rag document search tool
|
||||
settings.RAG_DOCUMENT_SEARCH_BACKEND = request.param
|
||||
settings.RAG_WEB_SEARCH_PROMPT_UPDATE = (
|
||||
"Based on the following document contents:\n\n{search_results}\n\n"
|
||||
"Please answer the user's question: {user_prompt}"
|
||||
)
|
||||
|
||||
settings.AI_BASE_URL = "https://www.external-ai-service.com/"
|
||||
settings.AI_API_KEY = "test-api-key"
|
||||
settings.AI_MODEL = "test-model"
|
||||
settings.AI_AGENT_INSTRUCTIONS = "You are a helpful test assistant :)"
|
||||
|
||||
# Albert API settings
|
||||
# Enable Albert API for document search
|
||||
settings.RAG_DOCUMENT_SEARCH_BACKEND = (
|
||||
"chat.agent_rag.document_rag_backends.albert_rag_backend.AlbertRagBackend"
|
||||
)
|
||||
settings.ALBERT_API_URL = "https://albert.api.etalab.gouv.fr"
|
||||
settings.ALBERT_API_KEY = "albert-api-key"
|
||||
|
||||
# Find API settings
|
||||
settings.FIND_API_URL = "https://find.api.example.com"
|
||||
settings.FIND_API_KEY = "find-api-key"
|
||||
settings.RAG_WEB_SEARCH_PROMPT_UPDATE = (
|
||||
"Based on the following document contents:\n\n{search_results}\n\n"
|
||||
"Please answer the user's question: {user_prompt}"
|
||||
)
|
||||
|
||||
return settings
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_refresh_access_token():
|
||||
"""Mock refresh_access_token to bypass token refresh in tests."""
|
||||
with mock.patch("utils.oidc.refresh_access_token") as mocked_refresh_access_token:
|
||||
session = SessionStore()
|
||||
session["oidc_access_token"] = "mocked-access-token"
|
||||
mocked_refresh_access_token.return_value = session
|
||||
yield mocked_refresh_access_token
|
||||
|
||||
|
||||
@pytest.fixture(name="sample_pdf_content")
|
||||
def fixture_sample_pdf_content():
|
||||
"""Create a dummy PDF content as BytesIO."""
|
||||
@@ -103,25 +81,17 @@ def fixture_sample_pdf_content():
|
||||
return BytesIO(pdf_data)
|
||||
|
||||
|
||||
@pytest.fixture(name="mock_document_api")
|
||||
def fixture_mock_document_api():
|
||||
@pytest.fixture(name="mock_albert_api")
|
||||
def fixture_mock_albert_api():
|
||||
"""Fixture to mock the Albert API endpoints."""
|
||||
# Mock collection creation
|
||||
|
||||
document_name = "sample.pdf"
|
||||
document_content = "This is the content of the PDF."
|
||||
prompt_tokens = 10
|
||||
completion_tokens = 20
|
||||
search_method = "semantic"
|
||||
search_score = 0.9
|
||||
|
||||
responses.post(
|
||||
"https://albert.api.etalab.gouv.fr/v1/collections",
|
||||
json={"id": "123", "name": "test-collection"},
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
# Mock Albert PDF parsing -> deprecated
|
||||
# Mock PDF parsing
|
||||
responses.post(
|
||||
"https://albert.api.etalab.gouv.fr/v1/parse-beta",
|
||||
json={
|
||||
@@ -131,7 +101,7 @@ def fixture_mock_document_api():
|
||||
"metadata": {"document_name": "sample.pdf"},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens},
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 20},
|
||||
},
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
@@ -149,42 +119,20 @@ def fixture_mock_document_api():
|
||||
json={
|
||||
"data": [
|
||||
{
|
||||
"method": search_method,
|
||||
"method": "semantic",
|
||||
"chunk": {
|
||||
"id": 123,
|
||||
"content": document_content,
|
||||
"metadata": {"document_name": document_name},
|
||||
"content": "This is the content of the PDF.",
|
||||
"metadata": {"document_name": "sample.pdf"},
|
||||
},
|
||||
"score": search_score,
|
||||
"score": 0.9,
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens},
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 20},
|
||||
},
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
# Mock document indexing (Find API)
|
||||
responses.post(
|
||||
"https://find.api.example.com/api/v1.0/documents/index/",
|
||||
json={"id": "456", "status": "indexed"},
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
# Mock document search (Find API)
|
||||
responses.post(
|
||||
"https://find.api.example.com/api/v1.0/documents/search/",
|
||||
json=[
|
||||
{
|
||||
"_source": {
|
||||
"title.fr": document_name,
|
||||
"content.fr": document_content,
|
||||
},
|
||||
"_score": search_score,
|
||||
}
|
||||
],
|
||||
status=status.HTTP_200_OK,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(name="mock_summarization_agent")
|
||||
def fixture_mock_summarization_agent():
|
||||
@@ -271,7 +219,7 @@ def fixture_mock_openai_stream():
|
||||
def test_post_conversation_with_document_upload(
|
||||
# pylint: disable=too-many-arguments,too-many-positional-arguments
|
||||
api_client,
|
||||
mock_document_api, # pylint: disable=unused-argument
|
||||
mock_albert_api, # pylint: disable=unused-argument
|
||||
sample_pdf_content,
|
||||
today_promt_date,
|
||||
mock_ai_agent_service,
|
||||
@@ -600,7 +548,7 @@ def test_post_conversation_with_document_upload_feature_disabled(
|
||||
@freeze_time()
|
||||
def test_post_conversation_with_document_upload_summarize( # pylint: disable=too-many-arguments,too-many-positional-arguments # noqa: PLR0913
|
||||
api_client,
|
||||
mock_document_api, # pylint: disable=unused-argument
|
||||
mock_albert_api, # pylint: disable=unused-argument
|
||||
sample_pdf_content,
|
||||
today_promt_date,
|
||||
mock_ai_agent_service,
|
||||
@@ -675,7 +623,7 @@ def test_post_conversation_with_document_upload_summarize( # pylint: disable=to
|
||||
'document discusses various topics."}\n'
|
||||
'0:"The document discusses various topics."\n'
|
||||
'f:{"messageId":"<mocked_uuid>"}\n'
|
||||
'd:{"finishReason":"stop","usage":{"promptTokens":283,"completionTokens":19}}\n'
|
||||
'd:{"finishReason":"stop","usage":{"promptTokens":287,"completionTokens":19}}\n'
|
||||
)
|
||||
|
||||
# Check that the conversation was updated
|
||||
|
||||
+2
-18
@@ -37,19 +37,11 @@ from chat.tests.utils import replace_uuids_with_placeholder
|
||||
pytestmark = pytest.mark.django_db(transaction=True)
|
||||
|
||||
|
||||
@pytest.fixture(
|
||||
autouse=True,
|
||||
params=[
|
||||
"chat.agent_rag.document_rag_backends.find_rag_backend.FindRagBackend",
|
||||
"chat.agent_rag.document_rag_backends.albert_rag_backend.AlbertRagBackend",
|
||||
],
|
||||
)
|
||||
def ai_settings(request, settings):
|
||||
@pytest.fixture(autouse=True)
|
||||
def ai_settings(settings):
|
||||
"""Fixture to set AI service URLs for testing."""
|
||||
settings.RAG_DOCUMENT_SEARCH_BACKEND = request.param
|
||||
settings.AI_BASE_URL = "https://www.external-ai-service.com/"
|
||||
settings.AI_API_KEY = "test-api-key"
|
||||
settings.FIND_API_KEY = "find-api-key"
|
||||
settings.AI_MODEL = "test-model"
|
||||
settings.AI_AGENT_INSTRUCTIONS = "You are a helpful test assistant :)"
|
||||
return settings
|
||||
@@ -93,10 +85,6 @@ def test_post_conversation_with_local_pdf_document_url(
|
||||
json={"id": "document_id", "object": "document"},
|
||||
status=200,
|
||||
)
|
||||
responses.post(
|
||||
"https://app-find/api/v1.0/documents/index/",
|
||||
status=200,
|
||||
)
|
||||
|
||||
chat_conversation = ChatConversationFactory(owner__language="en-us")
|
||||
api_client.force_authenticate(user=chat_conversation.owner)
|
||||
@@ -807,10 +795,6 @@ def test_post_conversation_with_local_not_pdf_document_url(
|
||||
json={"id": "document_id", "object": "document"},
|
||||
status=200,
|
||||
)
|
||||
responses.post(
|
||||
"https://app-find/api/v1.0/documents/index/",
|
||||
status=200,
|
||||
)
|
||||
|
||||
chat_conversation = ChatConversationFactory(owner__language="en-us")
|
||||
api_client.force_authenticate(user=chat_conversation.owner)
|
||||
|
||||
@@ -26,7 +26,7 @@ def add_document_rag_search_tool(agent: Agent) -> None:
|
||||
|
||||
document_store = document_store_backend(ctx.deps.conversation.collection_id)
|
||||
|
||||
rag_results = document_store.search(query, session=ctx.deps.session)
|
||||
rag_results = document_store.search(query)
|
||||
|
||||
ctx.usage += RunUsage(
|
||||
input_tokens=rag_results.usage.prompt_tokens,
|
||||
|
||||
@@ -101,7 +101,7 @@ async def document_summarize( # pylint: disable=too-many-locals
|
||||
)
|
||||
documents_chunks = chunker(
|
||||
[doc[1] for doc in documents],
|
||||
# overlap=settings.SUMMARIZATION_OVERLAP_SIZE,
|
||||
overlap=settings.SUMMARIZATION_OVERLAP_SIZE,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
|
||||
@@ -127,7 +127,7 @@ async def _extract_and_summarize_snippets_async(query: str, url: str) -> List[st
|
||||
return []
|
||||
|
||||
|
||||
async def _fetch_and_store_async(url: str, document_store, **kwargs) -> None:
|
||||
async def _fetch_and_store_async(url: str, document_store) -> None:
|
||||
"""Fetch, extract and store text content from the URL in the document store."""
|
||||
|
||||
try:
|
||||
@@ -136,7 +136,7 @@ async def _fetch_and_store_async(url: str, document_store, **kwargs) -> None:
|
||||
logger.debug("Fetched document: %s", document)
|
||||
|
||||
if document:
|
||||
await document_store.astore_document(url, document, **kwargs)
|
||||
await document_store.astore_document(url, document)
|
||||
except DocumentFetchError as e:
|
||||
logger.warning("Failed to fetch and store %s: %s", url, e)
|
||||
# Continue with other documents
|
||||
|
||||
@@ -7,13 +7,11 @@ from uuid import uuid4
|
||||
from django.conf import settings
|
||||
from django.core.files.storage import default_storage
|
||||
from django.http import Http404, StreamingHttpResponse
|
||||
from django.utils.decorators import method_decorator
|
||||
|
||||
import langfuse
|
||||
import magic
|
||||
import posthog
|
||||
from lasuite.malware_detection import malware_detection
|
||||
from lasuite.oidc_login.decorators import refresh_oidc_access_token
|
||||
from rest_framework import decorators, filters, mixins, permissions, status, viewsets
|
||||
from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError
|
||||
from rest_framework.response import Response
|
||||
@@ -124,7 +122,6 @@ class ChatViewSet( # pylint: disable=too-many-ancestors, abstract-method
|
||||
self.permission_classes = []
|
||||
return super().get_permissions()
|
||||
|
||||
@method_decorator(refresh_oidc_access_token)
|
||||
@decorators.action(
|
||||
methods=["post"],
|
||||
detail=True,
|
||||
@@ -176,7 +173,6 @@ class ChatViewSet( # pylint: disable=too-many-ancestors, abstract-method
|
||||
ai_service = AIAgentService(
|
||||
conversation=conversation,
|
||||
user=self.request.user,
|
||||
session=request.session,
|
||||
model_hrid=model_hrid,
|
||||
language=(
|
||||
self.request.user.language
|
||||
|
||||
@@ -22,7 +22,7 @@ def no_http_requests(monkeypatch):
|
||||
Credits: https://blog.jerrycodes.com/no-http-requests/
|
||||
"""
|
||||
|
||||
allowed_hosts = {"localhost", "127.0.0.1", "minio", "minio:9000"}
|
||||
allowed_hosts = {"localhost", "minio", "minio:9000"}
|
||||
original_urlopen = HTTPConnectionPool.urlopen
|
||||
|
||||
def urlopen_mock(self, method, url, *args, **kwargs):
|
||||
|
||||
@@ -841,23 +841,6 @@ USER QUESTION:
|
||||
environ_prefix=None,
|
||||
)
|
||||
|
||||
# Find
|
||||
FIND_API_KEY = values.Value(
|
||||
None,
|
||||
environ_name="FIND_API_KEY",
|
||||
environ_prefix=None,
|
||||
)
|
||||
FIND_API_URL = values.Value(
|
||||
"https://app-find/api",
|
||||
environ_name="FIND_API_URL",
|
||||
environ_prefix=None,
|
||||
)
|
||||
FIND_API_TIMEOUT = values.PositiveIntegerValue(
|
||||
default=30, # seconds
|
||||
environ_name="FIND_API_TIMEOUT",
|
||||
environ_prefix=None,
|
||||
)
|
||||
|
||||
# Logging
|
||||
# We want to make it easy to log to console but by default we log production
|
||||
# to Sentry and don't want to log to console.
|
||||
|
||||
@@ -43,9 +43,7 @@ dependencies = [
|
||||
"djangorestframework==3.16.1",
|
||||
"drf_spectacular==0.29.0",
|
||||
"dockerflow==2024.4.2",
|
||||
"docling",
|
||||
"easy_thumbnails==2.10.1",
|
||||
"easyocr",
|
||||
"factory_boy==3.3.3",
|
||||
"gunicorn==23.0.0",
|
||||
"jsonschema==4.25.1",
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
"""Utility functions for OIDC token management."""
|
||||
|
||||
from functools import wraps
|
||||
|
||||
from django.conf import settings
|
||||
|
||||
import requests
|
||||
from lasuite.oidc_login.backends import get_oidc_refresh_token, store_tokens
|
||||
from rest_framework.exceptions import AuthenticationFailed
|
||||
|
||||
|
||||
def refresh_access_token(session):
|
||||
"""Refresh the OIDC access token using the refresh token."""
|
||||
refresh_token = get_oidc_refresh_token(session)
|
||||
if not refresh_token:
|
||||
raise AuthenticationFailed({"error": "Refresh token is missing from session"})
|
||||
|
||||
response = requests.post(
|
||||
settings.OIDC_OP_TOKEN_ENDPOINT,
|
||||
data={
|
||||
"grant_type": "refresh_token",
|
||||
"client_id": settings.OIDC_RP_CLIENT_ID,
|
||||
"client_secret": settings.OIDC_RP_CLIENT_SECRET,
|
||||
"refresh_token": refresh_token,
|
||||
},
|
||||
timeout=5,
|
||||
)
|
||||
response.raise_for_status()
|
||||
token_info = response.json()
|
||||
|
||||
store_tokens(
|
||||
session,
|
||||
access_token=token_info.get("access_token"),
|
||||
id_token=None,
|
||||
refresh_token=token_info.get("refresh_token"),
|
||||
)
|
||||
return session
|
||||
|
||||
|
||||
def with_fresh_access_token(func):
|
||||
"""
|
||||
Decorator to handle OIDC token refresh and extraction.
|
||||
Expects 'session' in kwargs and update it with the fresh token.
|
||||
"""
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
session = kwargs.pop("session", None)
|
||||
if session is None:
|
||||
raise AuthenticationFailed({"error": "Session is required but not provided"})
|
||||
refreshed_session = refresh_access_token(session)
|
||||
return func(*args, session=refreshed_session, **kwargs)
|
||||
|
||||
return wrapper
|
||||
+1
-1
@@ -58,7 +58,7 @@ export const LeftPanelConversationItem = ({
|
||||
>
|
||||
<StyledLink
|
||||
href={`/chat/${conversation.id}/`}
|
||||
$css="overflow: auto; flex-grow: 1;"
|
||||
$css="overflow: auto; flex-grow: 1; color: var(--c--theme--colors--greyscale-900);"
|
||||
onClick={handleLinkClick}
|
||||
>
|
||||
<SimpleConversationItem showAccesses conversation={conversation} />
|
||||
|
||||
Reference in New Issue
Block a user