From 995f3f2ab754fa194cffdd62e1bcc1864bf8fb5d Mon Sep 17 00:00:00 2001 From: Quentin BEY Date: Fri, 21 Nov 2025 15:47:20 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9A=97=EF=B8=8F(evalap)=20add=20a=20simple?= =?UTF-8?q?=20OpenAI=20compatible=20endpoint=20to=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This provide a quick and dirty way to run evaluation on the Conversations assistant. --- docs/evaluation.md | 211 +++++++++++++++++++++++++++++ src/backend/core/urls.py | 8 ++ src/backend/evaluation/__init__.py | 0 src/backend/evaluation/views.py | 198 +++++++++++++++++++++++++++ 4 files changed, 417 insertions(+) create mode 100644 docs/evaluation.md create mode 100644 src/backend/evaluation/__init__.py create mode 100644 src/backend/evaluation/views.py diff --git a/docs/evaluation.md b/docs/evaluation.md new file mode 100644 index 0000000..a7abc45 --- /dev/null +++ b/docs/evaluation.md @@ -0,0 +1,211 @@ +# Evaluation + +To allow simple evaluation of language models, [EvalAP](https://github.com/etalab-ia/evalap/) provides an API +and a web interface to run evaluations on various datasets using different metrics. + +To allow to easily integrate EvalAP with Conversations a new endpoint "OpenAI compatible" is provided to call +the conversation agent. + + +> **Warning:** +> +> This is not really an Open AI compatible API, but it follows the same structure to make it easier +to use with existing tools. We only support simple inputs and outputs (no streaming, no function calls, etc). +The result returned will already have called the tools, etc. + +This endpoint is only available when running the stack locally (ie in "development" or "tests" mode) under the +`/v1/chat/completions` endpoint. + +See the backend's `evaluation/views.py` module for more details. + +## Conversations' configuration + +First you need to configure the backend for the experiment you want to run. For instance, if you want to compare +the Agent answer with and without a retrieval tool, you will need: + +- To create the demo data by running: + +```shell +$ make demo +``` + +- To update settings to the point to a new LLM configuration file, for instance in `env.d/development/common`: + +```ini +LLM_CONFIGURATION_FILE_PATH = /app/conversations/configuration/llm/evalap_experiments.json +``` + +And create the file `conversations/configuration/llm/evalap_experiments.json` with the following content: + +```json +{ + "models": [ + { + "hrid": "mistral-medium-2508-raw", + "model_name": "mistral-medium-2508", + "human_readable_name": "Mistral Medium 2508", + "provider_name": "mistral", + "profile": null, + "settings": {}, + "is_active": true, + "icon": null, + "system_prompt": "settings.AI_AGENT_INSTRUCTIONS", + "tools": [] + }, + { + "hrid": "mistral-medium-2508-with-web-search", + "model_name": "mistral-medium-2508", + "human_readable_name": "Mistral Medium 2508", + "provider_name": "mistral", + "profile": null, + "settings": {}, + "is_active": true, + "icon": null, + "system_prompt": "settings.AI_AGENT_INSTRUCTIONS", + "tools": ["web_search_brave"] + }, + { + "hrid": "default-summarization-model", + "model_name": "mistral-medium-2508", + "human_readable_name": "Mistral Medium 2508", + "provider_name": "mistral", + "profile": null, + "settings": {}, + "is_active": true, + "icon": null, + "system_prompt": "settings.SUMMARIZATION_SYSTEM_PROMPT", + "tools": [] + } + ], + "providers": [ + { + "hrid": "mistral", + "base_url": "https://api.mistral.ai/", + "api_key": "environ.MISTRAL_API_KEY", + "kind": "mistral" + } + ] +} +``` + +Which defines two models for the same LLM, one with a web search tool and one without. + +- Create the "SPECIFIC_RAG_DOCUMENT_SEARCH_TOOLS" if not already done: + +```ini +SPECIFIC_RAG_DOCUMENT_SEARCH_TOOLS={"tool_rag_french_public_services": {"collection_ids": [784, 785],"feature_flag_value": "disabled","tool_description": "Use this tool when the user asks for information about French public services, the French labor market, employment laws, social benefits, or assistance with administrative procedures."}} +``` + + +> **Note:** +> +> The specific tool configuration is not mandatory for evaluations, only if you want to +> test them. + +- Restart your stack to apply the changes: + +```shell +$ make run-backend +``` + +## EvalAP configuration + +You will need to configure EvalAP to call Conversations for chat completion. + +### Run the stack + +I needed to update the Docker compose file to add: + +```yaml + extra_hosts: + - "host.docker.internal:host-gateway" +``` + +Globally I followed the instructions in the EvalAP documentation, had a few issues with the stack initialization +but finally managed to run it with. + +### Create the dataset + +Read the EvalAP documentation to create a new dataset. I did a simple dataset with only two samples to check +the evaluation works. + +### Create the evaluation + +Same as before, read the EvalAP documentation to create a new evaluation. + +The important part is to configure the model to call Conversations, and use the extra parameters to +adapt feature flags if needed. + +```python +import requests + +# Replace with your Evalap API endpoint +API_URL = "http://localhost:8000/v1" + +# Replace with your API key or authentication token +HEADERS = { + "Authorization": "Bearer YOUR_API_KEY", + "Content-Type": "application/json" +} + +# Define your experiment set with CV schema +expset_name = "model_comparison_v1" +expset_readme = "Comparing performance of various LLMs on a QA dataset." +metrics = ["judge_precision", "output_length", "generation_time"] + +# Parameters common to all experiments +common_params = { + "dataset": "Dataset Bidon", # assuming this dataset has been added before + #"model": {"sampling_params": {"temperature": 0.2}}, + "metrics": metrics, + "judge_model": "albert-large", +} + +# Parameters that will vary across experiments +grid_params = { + "model": [ + { + "name": "etalab-plateform-mistral-medium-2508", + "aliased_name": "Mistral Medium", + # base_url points to Conversations API + "base_url": f"http://host.docker.internal:8071/v1", + "api_key": "plop", + "extra_params": { + "feature_flags": { + # Disable RAG tool + "tool_rag_french_public_services": "DISABLED", + }, + }, + }, + { + "name": "etalab-plateform-mistral-medium-2508", + "aliased_name": "Mistral Medium + RAG", + "base_url": f"http://host.docker.internal:8071/v1", + "api_key": "plop", + "extra_params": { + "feature_flags": { + # Enable RAG tool + "tool_rag_french_public_services": "ENABLED", + }, + }, + }, + ], +} + +# Create the experiment set with CV schema +expset = { + "name": expset_name, + "readme": expset_readme, + "cv": { + "common_params": common_params, + "grid_params": grid_params, + "repeat": 3 # Run each combination 3 times to measure variability + } +} + +# Launch the experiment set +requests.delete(f'{API_URL}/experiment_set/12', json=expset, headers=HEADERS) +response = requests.post(f'{API_URL}/experiment_set', json=expset, headers=HEADERS) +expset_id = response.json()["id"] +print(f"Experiment set {expset_id} is running") +``` diff --git a/src/backend/core/urls.py b/src/backend/core/urls.py index 33755fe..906d368 100644 --- a/src/backend/core/urls.py +++ b/src/backend/core/urls.py @@ -10,6 +10,7 @@ from core.api import viewsets from activation_codes import viewsets as activation_viewsets from chat.views import ChatConversationAttachmentViewSet, ChatViewSet, LLMConfigurationView +from evaluation.views import ChatCompletionsView # - Main endpoints router = DefaultRouter() @@ -41,3 +42,10 @@ urlpatterns = [ ), path(f"api/{settings.API_VERSION}/config/", viewsets.ConfigView.as_view()), ] + +if settings.ENVIRONMENT in ["development", "tests"]: + urlpatterns += [ + # Allow access to the OpenAI-like chat completions endpoint only in development and tests + # on http://localhost:8071/v1/chat/completions + path("v1/chat/completions", ChatCompletionsView.as_view(), name="chat_completions"), + ] diff --git a/src/backend/evaluation/__init__.py b/src/backend/evaluation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/backend/evaluation/views.py b/src/backend/evaluation/views.py new file mode 100644 index 0000000..79972ce --- /dev/null +++ b/src/backend/evaluation/views.py @@ -0,0 +1,198 @@ +""" +OpenAI-compatible API endpoint for AIAgentService. + +This module provides a /v1/chat/completions endpoint that translates +between OpenAI's API format and our AIAgentService. + +This is for evaluation purposes only and is not intended for production use. +Works with EvalAP (https://github.com/etalab-ia/evalap/tree/main) +""" + +import json +import time +import uuid +from typing import List, Optional + +from django.conf import settings +from django.http import JsonResponse +from django.utils.decorators import method_decorator +from django.views import View +from django.views.decorators.csrf import csrf_exempt + +from core.models import User + +from chat.ai_sdk_types import TextUIPart, UIMessage +from chat.clients.pydantic_ai import AIAgentService +from chat.models import ChatConversation +from chat.vercel_ai_sdk.core.events_v4 import ( + FinishMessagePart, + TextPart, +) + + +def create_openai_response( + response_id: str, + model: str, + content: str, + finish_reason: str = "stop", + usage: Optional[dict] = None, +) -> dict: + """ + Create an OpenAI-compatible non-streaming response. + + Note: we could use ChatCompletion from openai library. + """ + return { + "id": response_id, + "object": "chat.completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": content, + }, + "finish_reason": finish_reason, + } + ], + "usage": usage or {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + } + + +def openai_messages_to_ui_messages(openai_messages: List[dict]) -> List[UIMessage]: + """ + Convert OpenAI message format to UIMessage format for the backend view. + """ + ui_messages = [] + for msg in openai_messages: + role = msg.get("role", "user") + content = msg.get("content", "") + + # Handle content that can be string or list of content parts + if isinstance(content, list): + parts = [] + for part in content: + if part.get("type") == "text": + parts.append(TextUIPart(type="text", text=part.get("text", ""))) + # Add handling for images, etc. as needed + ui_messages.append( + UIMessage( + id=msg.get("id", str(uuid.uuid4())), + role=role, + parts=parts, + content="".join(content), + ) + ) + else: + ui_messages.append( + UIMessage( + id=msg.get("id", str(uuid.uuid4())), + role=role, + parts=[TextUIPart(type="text", text=content or "")], + content=content or "", + ) + ) + return ui_messages + + +@method_decorator(csrf_exempt, name="dispatch") +class ChatCompletionsView(View): + """ + OpenAI-compatible /v1/chat/completions endpoint. + + Usage: + POST /v1/chat/completions + { + "model": "your-model-id", + "messages": [{"role": "user", "content": "Hello!"}], + "stream": true, + "stream_options": {"include_usage": true} + } + """ + + async def post(self, request): + """ + Handle POST requests to the chat completions endpoint. + """ + if settings.ENVIRONMENT not in ["development", "tests"]: + return JsonResponse( + {"error": "This endpoint is for evaluation purposes only."}, status=403 + ) + + # Enforce the user + user = await User.objects.aget(email="conversations@conversations.world") + + # Parse request body to get parameters + try: + body = json.loads(request.body) + except json.JSONDecodeError: + return JsonResponse({"error": "Invalid JSON"}, status=400) + + # Extract parameters + model = body.get("model", "default-model") + messages = body.get("messages", []) + feature_flags = body.get("feature_flags", {}) + + # Monkey patch is_feature_enabled to use feature_flags from request + from core.feature_flags.helpers import ( # noqa: PLC0415 # pylint: disable=import-outside-toplevel + is_feature_enabled as original_is_feature_enabled, + ) + + def is_feature_enabled(tested_user: User, flag_name: str) -> bool: + if feature_flag := feature_flags.get(flag_name): + return feature_flag.upper() == "ENABLED" + return original_is_feature_enabled(tested_user, flag_name) + + import core.feature_flags.helpers # noqa: PLC0415 # pylint: disable=import-outside-toplevel + + core.feature_flags.helpers.is_feature_enabled = is_feature_enabled + + if not messages: + return JsonResponse( + {"error": {"message": "messages is required", "type": "invalid_request_error"}}, + status=400, + ) + + # Create a new conversation + conversation = await ChatConversation.objects.acreate( + owner=user, + messages=[], + pydantic_messages=[], + ) + + # Convert messages + ui_messages = openai_messages_to_ui_messages(messages) + + # Initialize service + service = AIAgentService( + conversation=conversation, + user=user, + model_hrid=model, + ) + + # Non-streaming response + full_content = "" + final_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0} + + async for event in service._run_agent(ui_messages): # noqa: SLF001 # pylint: disable=protected-access + if isinstance(event, TextPart): + full_content += event.text + elif isinstance(event, FinishMessagePart): + if event.usage: + final_usage = { + "prompt_tokens": event.usage.prompt_tokens, + "completion_tokens": event.usage.completion_tokens, + "total_tokens": event.usage.prompt_tokens + event.usage.completion_tokens, + } + + response_id = f"chatcmpl-{uuid.uuid4().hex[:24]}" + response = JsonResponse( + create_openai_response(response_id, model, full_content, "stop", final_usage) + ) + + # Remove the conversation to avoid accumulation + await conversation.adelete() + + return response