add responses API with examples

mlx_lm/SERVER.md

@@ -30,6 +30,8 @@ To see a full list of options run:
 mlx_lm.server --help
 ```
 
+## Chat completions API
+
 You can make a request to the model by running:
 
 ```shell
@@ -128,7 +130,23 @@ curl localhost:8080/v1/chat/completions \
     - `completion_tokens`: The number of tokens generated.
     - `total_tokens`: The total number of tokens, i.e. the sum of the above two fields.
 
-### List Models
+## Responses API
+
+The responses API follows the [OpenAI responses API
+spec](https://platform.openai.com/docs/quickstart?api-mode=responses)
+
+To make a request, use the `/reponses` endpoint. For exapmle:
+
+```shell
+curl localhost:8080/responses \
+  -H "Content-Type: application/json" \
+  -d '{
+     "input": [{"role": "user", "content": "Say this is a test!"}],
+     "temperature": 0.7
+   }'
+```
+
+## Models API
 
 Use the `v1/models` endpoint to list available models:
 

mlx_lm/examples/openai_responses.py

@@ -0,0 +1,59 @@
+# Copyright © 2025 Apple Inc.
+"""
+Examples using the OpenAI responses endpoint with mlx_lm.server.
+
+To run, first start the server:
+
+>>> mlx_lm.server
+
+Then run this script.
+
+More documentation on the API spec here:
+https://platform.openai.com/docs/quickstart?api-mode=responses
+"""
+from openai import OpenAI
+
+model = "mlx-community/Qwen3-4B-Instruct-2507-4bit"
+
+### Basic response example
+
+client = OpenAI(base_url="http://localhost:8080/v1", api_key="not-needed")
+
+response = client.responses.create(
+    model=model, input="Write a one-sentence bedtime story about a unicorn."
+)
+print(response.output_text)
+
+### Input with roles
+
+response = client.responses.create(
+    model=model,
+    input=[
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "input_text",
+                    "text": "Write a one-sentence bedtime story about a unicorn.",
+                },
+            ],
+        }
+    ],
+)
+print(response.output_text)
+
+### Streaming
+
+stream = client.responses.create(
+    model=model,
+    input=[
+        {
+            "role": "user",
+            "content": "Say 'double bubble bath' ten times fast.",
+        },
+    ],
+    stream=True,
+)
+
+for event in stream:
+    print(event)

mlx_lm/quant/dynamic_quant.py

@@ -230,7 +230,7 @@ def main():
         high_group_size=args.high_group_size,
     )
 
-    def quant_predicate(p, m, _):
+    def quant_predicate(p, m):
         if not hasattr(m, "to_quantized"):
             return False
         if sensitivities[p] > threshold:

mlx_lm/server.py

@@ -279,6 +279,8 @@ class APIHandler(BaseHTTPRequestHandler):
         endpoints = {
             "/v1/completions": self.handle_text_completions,
             "/v1/chat/completions": self.handle_chat_completions,
+            "/responses": self.handle_responses,
+            "/v1/responses": self.handle_responses,
             "/chat/completions": self.handle_chat_completions,
         }
 
@@ -328,6 +330,7 @@ class APIHandler(BaseHTTPRequestHandler):
             self.max_tokens = self.body.get(
                 "max_tokens", self.model_provider.cli_args.max_tokens
             )
+
         self.temperature = self.body.get(
             "temperature", self.model_provider.cli_args.temp
         )
@@ -492,6 +495,18 @@ class APIHandler(BaseHTTPRequestHandler):
                 "id": None,
             }
 
+        if self.stream and self.object_type == "response" and finish_reason is None:
+            return {
+                "type": "response.output_text.delta",
+                "delta": text,
+                # TODO, these need valid values
+                "sequence_number": None,
+                "item_id": None,
+                "output_index": 1,
+                "content_index": 0,
+                "logprobs": [],
+            }
+
         # Static response
         response = {
             "id": self.request_id,
@@ -499,14 +514,28 @@ class APIHandler(BaseHTTPRequestHandler):
             "object": self.object_type,
             "model": self.requested_model,
             "created": self.created,
-            "choices": [
-                {
-                    "index": 0,
-                    "finish_reason": finish_reason,
-                },
-            ],
         }
 
+        if self.object_type == "response":
+            response["output"] = [
+                {
+                    "type": "message",
+                    "role": "assistant",
+                    "content": [{"text": text, "type": "output_text"}],
+                }
+            ]
+            if self.stream:
+                return {"response": response, "type": "response.completed"}
+
+            return response
+
+        response["choices"] = [
+            {
+                "index": 0,
+                "finish_reason": finish_reason,
+            },
+        ]
+
         if token_logprobs or top_logprobs or tokens:
             response["choices"][0]["logprobs"] = {
                 "token_logprobs": token_logprobs,
@@ -864,6 +893,36 @@ class APIHandler(BaseHTTPRequestHandler):
 
         return prompt
 
+    def handle_responses(self) -> List[int]:
+        body = self.body
+        system_prompt = body.get("instructions")
+        prompt = body["input"]
+        tools = body.get("tools")
+        messages = []
+        if system_prompt:
+            messages = [{"role": "system", "content": system_prompt}]
+        if isinstance(prompt, list):
+            for message in prompt:
+                content = message["content"]
+                if isinstance(content, list):
+                    if len(content) != 1 or content[0]["type"] != "input_text":
+                        raise ValueError("Unsupported content type.")
+                    message["content"] = content[0]["text"]
+                messages.append(message)
+        else:
+            messages.append({"role": "user", "content": prompt})
+
+        # Determine response type
+        self.request_id = f"resp_{uuid.uuid4()}"
+        self.object_type = "response"
+        prompt = self.tokenizer.apply_chat_template(
+            messages,
+            tools=tools,
+            add_generation_prompt=True,
+            **self.model_provider.cli_args.chat_template_args,
+        )
+        return prompt
+
     def handle_text_completions(self) -> List[int]:
         """
         Handle a text completion request.

mlx_lm/utils.py

@@ -455,9 +455,7 @@ def quantize_model(
     group_size: int,
     bits: int,
     mode: str = "affine",
-    quant_predicate: Optional[
-        Callable[[str, nn.Module, dict], Union[bool, dict]]
-    ] = None,
+    quant_predicate: Optional[Callable[[str, nn.Module], Union[bool, dict]]] = None,
 ) -> Tuple[nn.Module, dict]:
     """
     Applies quantization to the model weights.
@@ -468,11 +466,10 @@ def quantize_model(
         group_size (int): Group size for quantization.
         bits (int): Bits per weight for quantization.
         mode (str): The quantization mode.
-        quant_predicate (Callable): A callable that decides how
-            to quantize each layer based on the path.
-            Accepts the layer `path`, the `module` and the model `config`.
-            Returns either a bool to signify quantize/no quantize or
-            a dict of quantization parameters to pass to `to_quantized`.
+        quant_predicate (Callable): A callable that decides how to quantize
+          each layer based on the path. Accepts the layer `path` and the
+          `module`. Returns either a bool to signify quantize/no quantize or
+          a dict of quantization parameters to pass to `to_quantized`.
 
     Returns:
         Tuple: Tuple containing quantized model and config.
@@ -496,7 +493,7 @@ def quantize_model(
             return False
         bool_or_params = True
         if quant_predicate is not None:
-            bool_or_params = quant_predicate(path, module, config)
+            bool_or_params = quant_predicate(path, module)
         if isinstance(bool_or_params, dict):
             quantized_config["quantization"][path] = bool_or_params
         elif fine_grained_config and bool_or_params: