fix: only shutdown runners that error unrecoverably

previously we would shutdown a runner that had previously failed,
leading to loops of create -> shutdown -> create.

now when a runner critically errors, we set its status to Shutdown, and
delete it. a failed runner is expected

.mlx_typings/mlx/core/__init__.pyi

@@ -2396,7 +2396,7 @@ def degrees(a: array, /, *, stream: Stream | Device | None = ...) -> array:
         array: The angles in degrees.
     """
 
-def depends(inputs: array | Sequence[array], dependencies: array | Sequence[array]):
+def depends[T](inputs: T, dependencies: array | Sequence[array]) -> T:
     """
     Insert dependencies between arrays in the graph. The outputs are
     identical to ``inputs`` but with dependencies on ``dependencies``.

.mlx_typings/mlx/nn/__init__.pyi

@@ -1,9 +1,5 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from layers import *
-from utils import *
+from .layers import *
+from .utils import *
 
 from . import init as init
 from . import losses as losses

.mlx_typings/mlx/nn/layers/__init__.pyi

@@ -1,20 +1,16 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from activations import *
-from base import *
-from containers import *
-from convolution import *
-from convolution_transpose import *
-from distributed import *
-from dropout import *
-from embedding import *
-from linear import *
-from normalization import *
-from pooling import *
-from positional_encoding import *
-from quantized import *
-from recurrent import *
-from transformer import *
-from upsample import *
+from .activations import *
+from .base import *
+from .containers import *
+from .convolution import *
+from .convolution_transpose import *
+from .distributed import *
+from .dropout import *
+from .embedding import *
+from .linear import *
+from .normalization import *
+from .pooling import *
+from .positional_encoding import *
+from .quantized import *
+from .recurrent import *
+from .transformer import *
+from .upsample import *

.mlx_typings/mlx/nn/layers/base.pyi

@@ -53,7 +53,7 @@ class Module(dict):
         mx.eval(model.parameters())
     """
 
-    __call__: Callable
+    def __call__(self, *args: Any, **kwargs: Any) -> mx.array: ...
     def __init__(self) -> None:
         """Should be called by the subclasses of ``Module``."""
 

.mlx_typings/mlx_lm/generate.pyi

@@ -30,7 +30,7 @@ def str2bool(string):  # -> bool:
 def setup_arg_parser():  # -> ArgumentParser:
     """Set up and return the argument parser."""
 
-generation_stream = ...
+generation_stream: mx.Stream
 
 @contextlib.contextmanager
 def wired_limit(
@@ -266,12 +266,12 @@ def _merge_caches(caches: Any) -> List[Any]: ...
 class Batch:
     uids: List[int]
     y: mx.array
-    logprobs: mx.array
+    logprobs: List[mx.array] | mx.array
     max_tokens: List[int]
     num_tokens: List[int]
     cache: List[Any]
-    samplers: List[Any]
-    logits_processors: List[Any]
+    samplers: List[Callable[[mx.array], mx.array] | None]
+    logits_processors: List[List[Callable[[mx.array, mx.array], mx.array]]]
     tokens: List[mx.array]
     def __len__(self) -> int: ...
     def filter(self, keep_idx: List[int]) -> None: ...
@@ -279,13 +279,18 @@ class Batch:
     def extract_cache(self, idx: int) -> List[Any]: ...
 
 class BatchGenerator:
-    model: Any
+    model: nn.Module
+    sampler: Callable[[mx.array], mx.array]
+    stop_tokens: set[int]
     max_kv_size: Optional[int]
     prefill_step_size: int
+    completion_batch_size: int
+    prefill_batch_size: int
     unprocessed_prompts: List[Any]
     active_batch: Optional[Batch]
     prompt_progress_callback: Callable[[List[Tuple[int, int, int]]], None]
     _stats: BatchStats
+    _next_count: int
 
     @dataclass
     class Response:

.mlx_typings/mlx_lm/models/cache.pyi

@@ -88,8 +88,8 @@ def create_attention_mask(
 ) -> array | Literal["causal"] | None: ...
 
 class _BaseCache(Cache):
-    keys: mx.array
-    values: mx.array
+    keys: mx.array | None
+    values: mx.array | None
     offset: int
     @property
     def state(self) -> tuple[mx.array | None, mx.array | None]: ...
@@ -268,29 +268,14 @@ class CacheList(_BaseCache):
         """
 
 class BatchKVCache(_BaseCache):
-    step = ...
-    def __init__(self, left_padding: List[int]) -> None:
-        """
-        The BatchKV cache expects inputs to be left-padded.
-
-        E.g. the following prompts:
-
-            [1, 3, 5]
-            [7]
-            [2, 6, 8, 9]
-
-        Should be padded like so:
-
-            [0, 1, 3, 5]
-            [0, 0, 0, 7]
-            [2, 6, 8, 9]
-
-        And ``left_padding`` specifies the amount of padding for each.
-        In this case, ``left_padding = [1, 3, 0]``.
-        """
-
-    def update_and_fetch(self, keys, values):  # -> tuple[array | Any, array | Any]:
-        ...
+    step: int
+    keys: array | None
+    values: array | None
+    offset: array
+    left_padding: array
+    _idx: int
+    def __init__(self, left_padding: List[int]) -> None: ...
+    def update_and_fetch(self, keys: array, values: array) -> tuple[array, array]: ...
     @property
     def state(
         self,
@@ -316,12 +301,21 @@ class BatchKVCache(_BaseCache):
         """
 
 class BatchRotatingKVCache(_BaseCache):
-    step = ...
-    def __init__(self, max_size, left_padding: List[int]) -> None: ...
-    def update_and_fetch(
-        self, keys, values
-    ):  # -> tuple[array | Any, array | Any] | tuple[array | Any, array | Any | None]:
-        ...
+    step: int
+    keys: array | None
+    values: array | None
+    offset: array
+    left_padding: array
+    max_size: int
+    _idx: int
+    _offset: int
+    rotated: bool
+    _lengths: array | None
+    def __init__(self, max_size: int, left_padding: List[int]) -> None: ...
+    def _trim(self, trim_size: int, v: array, append: array | None = ...) -> array: ...
+    def _update_in_place(self, keys: array, values: array) -> tuple[array, array]: ...
+    def _update_concat(self, keys: array, values: array) -> tuple[array, array]: ...
+    def update_and_fetch(self, keys: array, values: array) -> tuple[array, array]: ...
     @property
     def state(
         self,

.mlx_typings/mlx_lm/models/gated_delta.pyi

@@ -0,0 +1,35 @@
+from typing import Optional
+
+import mlx.core as mx
+
+def compute_g(A_log: mx.array, a: mx.array, dt_bias: mx.array) -> mx.array: ...
+def gated_delta_update(
+    q: mx.array,
+    k: mx.array,
+    v: mx.array,
+    a: mx.array,
+    b: mx.array,
+    A_log: mx.array,
+    dt_bias: mx.array,
+    state: Optional[mx.array] = ...,
+    mask: Optional[mx.array] = ...,
+    use_kernel: bool = ...,
+) -> tuple[mx.array, mx.array]: ...
+def gated_delta_ops(
+    q: mx.array,
+    k: mx.array,
+    v: mx.array,
+    g: mx.array,
+    beta: mx.array,
+    state: Optional[mx.array] = ...,
+    mask: Optional[mx.array] = ...,
+) -> tuple[mx.array, mx.array]: ...
+def gated_delta_kernel(
+    q: mx.array,
+    k: mx.array,
+    v: mx.array,
+    g: mx.array,
+    beta: mx.array,
+    state: mx.array,
+    mask: Optional[mx.array] = ...,
+) -> tuple[mx.array, mx.array]: ...

.mlx_typings/mlx_lm/models/rope_utils.pyi

@@ -0,0 +1,51 @@
+from typing import Any, Optional
+
+import mlx.nn as nn
+
+class YarnRoPE(nn.Module):
+    def __init__(
+        self,
+        dims: int,
+        traditional: bool = ...,
+        max_position_embeddings: int = ...,
+        base: float = ...,
+        scaling_factor: float = ...,
+        original_max_position_embeddings: int = ...,
+        beta_fast: float = ...,
+        beta_slow: float = ...,
+        mscale: float = ...,
+        mscale_all_dim: float = ...,
+    ) -> None: ...
+
+class Llama3RoPE(nn.Module):
+    def __init__(
+        self,
+        dims: int,
+        traditional: bool = ...,
+        max_position_embeddings: int = ...,
+        base: float = ...,
+        scaling_factor: float = ...,
+        original_max_position_embeddings: int = ...,
+        low_freq_factor: float = ...,
+        high_freq_factor: float = ...,
+    ) -> None: ...
+
+class SuScaledRoPE(nn.Module):
+    def __init__(
+        self,
+        dims: int,
+        traditional: bool = ...,
+        max_position_embeddings: int = ...,
+        base: float = ...,
+        short_factor: Any = ...,
+        long_factor: Any = ...,
+        original_max_position_embeddings: int = ...,
+    ) -> None: ...
+
+def initialize_rope(
+    dims: int,
+    base: float = ...,
+    traditional: bool = ...,
+    scaling_config: Optional[dict[str, Any]] = ...,
+    max_position_embeddings: Optional[int] = ...,
+) -> nn.Module: ...

CONTRIBUTING.md

@@ -11,9 +11,18 @@ To run EXO from source:
   ```bash
   brew install uv
   ```
-- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
   ```bash
-  brew install macmon
+  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+  rustup toolchain install nightly
+  ```
+- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+  Use the pinned fork revision used by this repo instead of Homebrew `macmon`.
+  ```bash
+  cargo install --git https://github.com/swiftraccoon/macmon \
+    --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
+    macmon \
+    --force
   ```
 
 ```bash

README.md

@@ -95,11 +95,10 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
   ```
 - [uv](https://github.com/astral-sh/uv) (for Python dependency management)
-- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
 - [node](https://github.com/nodejs/node) (for building the dashboard)
 
   ```bash
-  brew install uv macmon node
+  brew install uv node
   ```
 - [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
 
@@ -107,6 +106,17 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
   rustup toolchain install nightly
   ```
+- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+
+  Install the pinned fork revision used by this repo instead of Homebrew `macmon`.
+  Homebrew `macmon 0.6.1` still crashes on Apple M5.
+
+  ```bash
+  cargo install --git https://github.com/swiftraccoon/macmon \
+    --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
+    macmon \
+    --force
+  ```
 
 Clone the repo, build the dashboard, and run exo:
 

app/EXO/EXO/ExoProcessController.swift

@@ -4,6 +4,7 @@ import Foundation
 
 private let customNamespaceKey = "EXOCustomNamespace"
 private let hfTokenKey = "EXOHFToken"
+private let hfEndpointKey = "EXOHFEndpoint"
 private let enableImageModelsKey = "EXOEnableImageModels"
 private let offlineModeKey = "EXOOfflineMode"
 private let onboardingCompletedKey = "EXOOnboardingCompleted"
@@ -53,6 +54,14 @@ final class ExoProcessController: ObservableObject {
             UserDefaults.standard.set(hfToken, forKey: hfTokenKey)
         }
     }
+    @Published var hfEndpoint: String = {
+        return UserDefaults.standard.string(forKey: hfEndpointKey) ?? ""
+    }()
+    {
+        didSet {
+            UserDefaults.standard.set(hfEndpoint, forKey: hfEndpointKey)
+        }
+    }
     @Published var enableImageModels: Bool = {
         return UserDefaults.standard.bool(forKey: enableImageModelsKey)
     }()
@@ -273,6 +282,9 @@ final class ExoProcessController: ObservableObject {
         if !hfToken.isEmpty {
             environment["HF_TOKEN"] = hfToken
         }
+        if !hfEndpoint.isEmpty {
+            environment["HF_ENDPOINT"] = hfEndpoint
+        }
         if enableImageModels {
             environment["EXO_ENABLE_IMAGE_MODELS"] = "true"
         }

app/EXO/EXO/Views/SettingsView.swift

@@ -12,6 +12,7 @@ struct SettingsView: View {
 
     @State private var pendingNamespace: String = ""
     @State private var pendingHFToken: String = ""
+    @State private var pendingHFEndpoint: String = ""
     @State private var pendingEnableImageModels = false
     @State private var pendingOfflineMode = false
     @State private var needsRestart = false
@@ -42,6 +43,7 @@ struct SettingsView: View {
         .onAppear {
             pendingNamespace = controller.customNamespace
             pendingHFToken = controller.hfToken
+            pendingHFEndpoint = controller.hfEndpoint
             pendingEnableImageModels = controller.enableImageModels
             pendingOfflineMode = controller.offlineMode
             needsRestart = false
@@ -74,6 +76,17 @@ struct SettingsView: View {
                     .foregroundColor(.secondary)
             }
 
+            Section {
+                LabeledContent("HuggingFace Endpoint") {
+                    TextField("default", text: $pendingHFEndpoint)
+                        .textFieldStyle(.roundedBorder)
+                        .frame(width: 200)
+                }
+                Text("Defaults to huggingface.co. Use a mirror (e.g. hf-mirror.com) for China.")
+                    .font(.caption)
+                    .foregroundColor(.secondary)
+            }
+
             Section {
                 Toggle("Offline Mode", isOn: $pendingOfflineMode)
                 Text("Skip internet checks and use only locally available models.")
@@ -454,6 +467,7 @@ struct SettingsView: View {
 
     private var hasGeneralChanges: Bool {
         pendingNamespace != controller.customNamespace || pendingHFToken != controller.hfToken
+            || pendingHFEndpoint != controller.hfEndpoint
             || pendingOfflineMode != controller.offlineMode
     }
 
@@ -464,6 +478,7 @@ struct SettingsView: View {
     private func applyGeneralSettings() {
         controller.customNamespace = pendingNamespace
         controller.hfToken = pendingHFToken
+        controller.hfEndpoint = pendingHFEndpoint
         controller.offlineMode = pendingOfflineMode
         restartIfRunning()
     }

bench/exo_bench.py

@@ -501,23 +501,21 @@ def main() -> int:
                                     for x, _ in batch_results
                                     if x["stats"]["generation_tps"] > 0
                                 ]
-                                agg_gen_tps = (
+                                per_req_tps = (
                                     mean(valid_gen_tps) if valid_gen_tps else 0.0
                                 )
-                                gen_tps = agg_gen_tps / concurrency
+                                agg_gen_tps = per_req_tps * concurrency
                                 logger.info(
                                     f"[concurrent {concurrency}x]  "
                                     f"agg_gen_tps={agg_gen_tps:.2f}  "
-                                    f"gen_tps={gen_tps:.2f}  "
+                                    f"per_req_tps={per_req_tps:.2f}  "
                                     f"errors={batch_errors}"
                                 )
 
                     if runs:
                         prompt_tps = mean(x["stats"]["prompt_tps"] for x in runs)
-                        gen_tps = mean(
-                            x["stats"]["generation_tps"] / x["concurrency"]
-                            for x in runs
-                        )
+                        per_req_tps = mean(x["stats"]["generation_tps"] for x in runs)
+                        gen_tps = per_req_tps * concurrency
                         ptok = mean(x["stats"]["prompt_tokens"] for x in runs)
                         gtok = mean(x["stats"]["generation_tokens"] for x in runs)
                         peak = mean(

dashboard/src/lib/components/FamilyLogos.svelte

@@ -88,6 +88,12 @@
       d="M22.012 0h1.032v.927H24v.968h-.956V3.78h-1.032V1.896h-1.878v-.97h1.878V0zM2.6 12.371V1.87h.969v10.502h-.97zm10.423.66h10.95v.918h-6.208v9.579h-4.742V13.03zM5.629 3.333v12.356H0v4.51h10.386V8L20.859 8l-.003-4.668-15.227.001z"
     />
   </svg>
+{:else if family === "nemotron"}
+  <svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
+    <path
+      d="M8.948 8.798v-1.43a6.7 6.7 0 0 1 .424-.018c3.922-.124 6.493 3.374 6.493 3.374s-2.774 3.851-5.75 3.851c-.398 0-.787-.062-1.158-.185v-4.346c1.528.185 1.837.857 2.747 2.385l2.04-1.714s-1.492-1.952-4-1.952a6.016 6.016 0 0 0-.796.035m0-4.735v2.138l.424-.027c5.45-.185 9.01 4.47 9.01 4.47s-4.08 4.964-8.33 4.964c-.37 0-.733-.035-1.095-.097v1.325c.3.035.61.062.91.062 3.957 0 6.82-2.023 9.593-4.408.459.371 2.34 1.263 2.73 1.652-2.633 2.208-8.772 3.984-12.253 3.984-.335 0-.653-.018-.971-.053v1.864H24V4.063zm0 10.326v1.131c-3.657-.654-4.673-4.46-4.673-4.46s1.758-1.944 4.673-2.262v1.237H8.94c-1.528-.186-2.73 1.245-2.73 1.245s.68 2.412 2.739 3.11M2.456 10.9s2.164-3.197 6.5-3.533V6.201C4.153 6.59 0 10.653 0 10.653s2.35 6.802 8.948 7.42v-1.237c-4.84-.6-6.492-5.936-6.492-5.936z"
+    />
+  </svg>
 {:else}
   <svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
     <path

dashboard/src/lib/components/FamilySidebar.svelte

@@ -31,6 +31,7 @@
     kimi: "Kimi",
     flux: "FLUX",
     "qwen-image": "Qwen Img",
+    nemotron: "NVIDIA",
   };
 
   function getFamilyName(family: string): string {

dashboard/src/lib/components/ModelCard.svelte

@@ -16,7 +16,9 @@
       perNode?: Array<{
         nodeId: string;
         nodeName: string;
-        progress: DownloadProgress;
+        status: "completed" | "partial" | "pending" | "downloading";
+        percentage: number;
+        progress: DownloadProgress | null;
       }>;
     } | null;
     nodes?: Record<string, NodeInfo>;
@@ -145,10 +147,7 @@
     return `${s}s`;
   }
 
-  const isDownloading = $derived(downloadStatus?.isDownloading ?? false);
-  const progress = $derived(downloadStatus?.progress);
-  const percentage = $derived(progress?.percentage ?? 0);
-  let expandedNodes = $state<Set<string>>(new Set());
+  const perNode = $derived(downloadStatus?.perNode ?? []);
 
   function toggleNodeDetails(nodeId: string): void {
     const next = new Set(expandedNodes);
@@ -587,23 +586,49 @@
       </span>
     </div>
 
-    <!-- Download Status -->
-    {#if isDownloading && progress}
+    <!-- Download Status (per-node) -->
+    {#if perNode.length > 0}
       <div class="mb-2 space-y-1">
-        <div class="flex items-center justify-between text-xs font-mono">
-          <span class="text-blue-400 tracking-wider uppercase">Downloading</span
-          >
-          <span class="text-white/60"
-            >{percentage.toFixed(1)}% &middot; {formatSpeed(progress.speed)}
-            &middot; {formatEta(progress.etaMs)}</span
-          >
-        </div>
-        <div class="h-1 bg-exo-medium-gray/30 rounded overflow-hidden">
-          <div
-            class="h-full bg-blue-500/70 transition-all duration-300"
-            style="width: {percentage}%"
-          ></div>
+        <div
+          class="text-[10px] font-mono text-white/20 tracking-widest uppercase"
+        >
+          Download progress
         </div>
+        {#each perNode as node}
+          <div class="flex items-center gap-2 text-xs font-mono">
+            <span class="text-white/40 w-20 truncate" title={node.nodeId}
+              >{node.nodeName}</span
+            >
+            <div
+              class="flex-1 h-1 bg-exo-medium-gray/30 rounded overflow-hidden"
+            >
+              <div
+                class="h-full transition-all duration-300 {node.status ===
+                'downloading'
+                  ? 'bg-blue-500/70'
+                  : node.status === 'completed'
+                    ? 'bg-exo-yellow/40'
+                    : 'bg-white/20'}"
+                style="width: {node.percentage}%"
+              ></div>
+            </div>
+            <span
+              class="text-right {node.status === 'completed'
+                ? 'text-exo-yellow/60'
+                : node.status === 'downloading'
+                  ? 'text-blue-400/60'
+                  : 'text-white/30'}"
+            >
+              {#if node.status === "downloading" && node.progress}
+                {Math.round(node.percentage)}% {formatSpeed(
+                  node.progress.speed,
+                )}
+              {:else}
+                {node.percentage > 0 ? `${Math.round(node.percentage)}%` : "0%"}
+              {/if}
+            </span>
+          </div>
+        {/each}
       </div>
     {/if}
 
@@ -662,15 +687,7 @@
             {@const allConnections =
               isDebugMode && usedNodes.length > 1
                 ? (() => {
-                    const conns: Array<{
-                      ip: string;
-                      iface: string | null;
-                      from: string;
-                      to: string;
-                      midX: number;
-                      midY: number;
-                      arrow: string;
-                    }> = [];
+                    const conns: Array = [];
                     for (let i = 0; i < usedNodes.length; i++) {
                       for (let j = i + 1; j < usedNodes.length; j++) {
                         const n1 = usedNodes[i];
@@ -682,7 +699,12 @@
                           const toPos = nodePositions[c.to];
                           const arrow =
                             fromPos && toPos ? getArrow(fromPos, toPos) : "→";
-                          conns.push({ ...c, midX, midY, arrow });
+                          conns.push({
+                            ...c,
+                            midX,
+                            midY,
+                            arrow,
+                          });
                         }
                       }
                     }

dashboard/src/lib/components/ModelPickerModal.svelte

@@ -459,6 +459,7 @@
       "llama",
       "flux",
       "qwen-image",
+      "nemotron",
     ];
     return Array.from(families).sort((a, b) => {
       const aIdx = familyOrder.indexOf(a);

dashboard/src/lib/stores/app.svelte.ts

@@ -1793,6 +1793,14 @@ class AppStore {
             this.persistConversation(targetConversationId);
           }
         },
+        {
+          generation_stats: (data) => {
+            const stats = data as { generation_tps: number };
+            if (stats.generation_tps > 0) {
+              this.tps = stats.generation_tps;
+            }
+          },
+        },
       );
 
       // Final update
@@ -1990,6 +1998,14 @@ class AppStore {
             this.persistConversation(targetConversationId);
           }
         },
+        {
+          generation_stats: (data) => {
+            const stats = data as { generation_tps: number };
+            if (stats.generation_tps > 0) {
+              this.tps = stats.generation_tps;
+            }
+          },
+        },
       );
 
       // Final cleanup of the message (if conversation still exists)
@@ -2397,7 +2413,7 @@ class AppStore {
 
       let streamedContent = "";
       let streamedThinking = "";
-
+      let serverTpsReceived = false;
       interface ChatCompletionChunk {
         choices?: Array<{
           delta?: { content?: string; reasoning_content?: string };
@@ -2462,7 +2478,6 @@ class AppStore {
             tokenCount += 1;
             this.totalTokens = tokenCount;
 
-            // Update real-time TPS during streaming
             if (firstTokenTime !== null && tokenCount > 1) {
               const elapsed = performance.now() - firstTokenTime;
               this.tps = (tokenCount / elapsed) * 1000;
@@ -2513,16 +2528,24 @@ class AppStore {
               startedAt: this.prefillProgress?.startedAt ?? performance.now(),
             };
           },
+          generation_stats: (data) => {
+            const stats = data as { generation_tps: number };
+
+            if (stats.generation_tps > 0) {
+              this.tps = stats.generation_tps;
+              serverTpsReceived = true;
+            }
+          },
         },
       );
 
       // Clear prefill progress after stream ends
       this.prefillProgress = null;
 
-      // Calculate final TPS
-      if (firstTokenTime !== null && tokenCount > 1) {
+      // Use server-side TPS if available, otherwise fall back to client-side
+      if (!serverTpsReceived && firstTokenTime !== null && tokenCount > 1) {
         const totalGenerationTime = performance.now() - firstTokenTime;
-        this.tps = (tokenCount / totalGenerationTime) * 1000; // tokens per second
+        this.tps = (tokenCount / totalGenerationTime) * 1000;
       }
 
       // Final cleanup of the message (if conversation still exists)

dashboard/src/routes/+page.svelte

@@ -42,6 +42,7 @@
     setSelectedChatModel,
     selectedChatModel,
     sendMessage,
+    thinkingEnabled,
     generateImage,
     editImage,
     editingImage,
@@ -852,7 +853,7 @@
   ) {
     const model = selectedChatModel();
     if (!model) {
-      sendMessage(content, files, null);
+      sendMessage(content, files, thinkingEnabled());
       return;
     }
 
@@ -880,7 +881,7 @@
     }
 
     // Default: text chat
-    sendMessage(content, files, null);
+    sendMessage(content, files, thinkingEnabled());
   }
 
   let selectedSharding = $state<"Pipeline" | "Tensor">("Pipeline");
@@ -1535,34 +1536,44 @@
   }
 
   // Helper to get download status for a model (checks all downloads for matching model ID)
-  function getModelDownloadStatus(modelId: string): {
+  type NodeDownloadStatus = {
+    nodeId: string;
+    nodeName: string;
+    status: "completed" | "partial" | "pending" | "downloading";
+    percentage: number;
+    progress: DownloadProgress | null;
+  };
+
+  // Shared helper: collect per-node download status for a model across a set of nodes.
+  // Handles deduplication, entry parsing, and aggregation in one place.
+  function collectDownloadStatus(
+    modelId: string,
+    nodeIds?: string[],
+  ): {
     isDownloading: boolean;
     progress: DownloadProgress | null;
-    perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }>;
+    perNode: NodeDownloadStatus[];
+    failedError: string | null;
   } {
+    const empty = {
+      isDownloading: false,
+      progress: null,
+      perNode: [] as NodeDownloadStatus[],
+      failedError: null,
+    };
+
     if (!downloadsData || Object.keys(downloadsData).length === 0) {
-      return { isDownloading: false, progress: null, perNode: [] };
+      return empty;
     }
 
-    let totalBytes = 0;
-    let downloadedBytes = 0;
-    let totalSpeed = 0;
-    let completedFiles = 0;
-    let totalFiles = 0;
-    let isDownloading = false;
-    const allFiles: DownloadProgress["files"] = [];
-    const perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }> = [];
+    // Deduplicate by nodeId — a node can have multiple entries for the same model
+    // (e.g. PipelineShardMetadata + TensorShardMetadata). Keep the last entry,
+    // which is the most recently applied event.
+    const perNodeMap = new Map<string, NodeDownloadStatus>();
 
-    // Check all nodes for downloads matching this model
+    const nodeIdSet = nodeIds ? new Set(nodeIds) : null;
     for (const [nodeId, nodeDownloads] of Object.entries(downloadsData)) {
+      if (nodeIdSet && !nodeIdSet.has(nodeId)) continue;
       if (!Array.isArray(nodeDownloads)) continue;
 
       for (const downloadWrapped of nodeDownloads) {
@@ -1575,29 +1586,45 @@
         const downloadPayload = (downloadWrapped as Record<string, unknown>)[
           downloadKind
         ] as Record<string, unknown>;
-
-        if (
-          downloadKind !== "DownloadOngoing" &&
-          downloadKind !== "DownloadPending"
-        )
-          continue;
         if (!downloadPayload) continue;
 
         const downloadModelId = extractModelIdFromDownload(downloadPayload);
+        if (!downloadModelId || downloadModelId !== modelId) continue;
 
-        // Match if the model ID contains or equals the requested model
-        // (handles cases like "mlx-community/Meta-Llama..." matching)
-        if (
-          !downloadModelId ||
-          !downloadModelId.includes(modelId.split("/").pop() || modelId)
-        ) {
-          // Try exact match or partial match
-          if (downloadModelId !== modelId) continue;
+        // DownloadFailed — return with any data collected so far
+        if (downloadKind === "DownloadFailed") {
+          return {
+            isDownloading: false,
+            progress: null,
+            perNode: Array.from(perNodeMap.values()),
+            failedError:
+              (downloadPayload.errorMessage as string) ||
+              (downloadPayload.error_message as string) ||
+              "Download failed",
+          };
+        }
+
+        if (
+          downloadKind !== "DownloadOngoing" &&
+          downloadKind !== "DownloadPending" &&
+          downloadKind !== "DownloadCompleted"
+        )
+          continue;
+
+        const nodeName =
+          data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
+
+        if (downloadKind === "DownloadCompleted") {
+          perNodeMap.set(nodeId, {
+            nodeId,
+            nodeName,
+            status: "completed",
+            percentage: 100,
+            progress: null,
+          });
+          continue;
         }
 
-        // For DownloadPending with partial bytes (paused/resumed downloads),
-        // synthesize a progress object from the top-level downloaded/total fields
-        let progress: DownloadProgress | null;
         if (downloadKind === "DownloadPending") {
           const pendingDownloaded = getBytes(
             downloadPayload.downloaded ??
@@ -1610,44 +1637,67 @@
               downloadPayload.totalBytes,
           );
           if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
-          isDownloading = true;
-          progress = {
-            totalBytes: pendingTotal,
-            downloadedBytes: pendingDownloaded,
-            speed: 0,
-            etaMs: 0,
-            percentage:
-              pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
-            completedFiles: 0,
-            totalFiles: 0,
-            files: [],
-          };
-        } else {
-          isDownloading = true;
-          progress = parseDownloadProgress(downloadPayload);
+          const pct =
+            pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0;
+          perNodeMap.set(nodeId, {
+            nodeId,
+            nodeName,
+            status: pendingDownloaded > 0 ? "partial" : "pending",
+            percentage: pct,
+            progress: null,
+          });
+          continue;
         }
 
-        if (progress) {
-          // Sum all values across nodes - each node downloads independently
-          totalBytes += progress.totalBytes;
-          downloadedBytes += progress.downloadedBytes;
-          totalSpeed += progress.speed;
-          completedFiles += progress.completedFiles;
-          totalFiles += progress.totalFiles;
-          allFiles.push(...progress.files);
+        // DownloadOngoing
+        const progress = parseDownloadProgress(downloadPayload);
+        if (
+          !progress ||
+          (progress.downloadedBytes <= 0 && progress.totalBytes <= 0)
+        )
+          continue;
 
-          const nodeName =
-            data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
-          perNode.push({ nodeId, nodeName, progress });
-        }
+        perNodeMap.set(nodeId, {
+          nodeId,
+          nodeName,
+          status: "downloading",
+          percentage: progress.percentage,
+          progress,
+        });
+      }
+    }
+
+    // Aggregate from deduplicated per-node entries
+    const perNode = Array.from(perNodeMap.values());
+    let totalBytes = 0;
+    let downloadedBytes = 0;
+    let totalSpeed = 0;
+    let completedFiles = 0;
+    let totalFiles = 0;
+    let isDownloading = false;
+    const allFiles: DownloadProgress["files"] = [];
+
+    for (const node of perNode) {
+      if (node.status === "downloading" && node.progress) {
+        isDownloading = true;
+        totalBytes += node.progress.totalBytes;
+        downloadedBytes += node.progress.downloadedBytes;
+        totalSpeed += node.progress.speed;
+        completedFiles += node.progress.completedFiles;
+        totalFiles += node.progress.totalFiles;
+        allFiles.push(...node.progress.files);
       }
     }
 
     if (!isDownloading) {
-      return { isDownloading: false, progress: null, perNode: [] };
+      return {
+        isDownloading: false,
+        progress: null,
+        perNode,
+        failedError: null,
+      };
     }
 
-    // ETA = total remaining bytes / total speed across all nodes
     const remainingBytes = totalBytes - downloadedBytes;
     const etaMs = totalSpeed > 0 ? (remainingBytes / totalSpeed) * 1000 : 0;
 
@@ -1664,9 +1714,21 @@
         files: allFiles,
       },
       perNode,
+      failedError: null,
     };
   }
 
+  function getModelDownloadStatus(
+    modelId: string,
+    nodeIds?: string[],
+  ): {
+    isDownloading: boolean;
+    progress: DownloadProgress | null;
+    perNode: NodeDownloadStatus[];
+  } {
+    return collectDownloadStatus(modelId, nodeIds);
+  }
+
   // Helper to get download status for an instance
   function getInstanceDownloadStatus(
     instanceId: string,
@@ -1677,26 +1739,9 @@
     errorMessage: string | null;
     progress: DownloadProgress | null;
     statusText: string;
-    perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }>;
+    perNode: NodeDownloadStatus[];
   } {
-    if (!downloadsData || Object.keys(downloadsData).length === 0) {
-      // No download data yet — defer to runner status instead of assuming RUNNING
-      const statusInfo = deriveInstanceStatus(instanceWrapped);
-      return {
-        isDownloading: false,
-        isFailed: false,
-        errorMessage: null,
-        progress: null,
-        statusText: statusInfo.statusText,
-        perNode: [],
-      };
-    }
-
-    // Unwrap the instance
+    // Unwrap the instance to get shard assignments
     const [instanceTag, instance] = getTagged(instanceWrapped);
     if (!instance || typeof instance !== "object") {
       return {
@@ -1716,132 +1761,9 @@
         modelId?: string;
       };
     };
-    const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
-    const runnerToShard = inst.shardAssignments?.runnerToShard || {};
     const instanceModelId = inst.shardAssignments?.modelId;
 
-    // Build reverse mapping: runnerId -> nodeId
-    const runnerToNode: Record<string, string> = {};
-    for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) {
-      runnerToNode[runnerId] = nodeId;
-    }
-
-    let totalBytes = 0;
-    let downloadedBytes = 0;
-    let totalSpeed = 0;
-    let completedFiles = 0;
-    let totalFiles = 0;
-    let isDownloading = false;
-    const allFiles: DownloadProgress["files"] = [];
-    const perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }> = [];
-
-    // Check downloads for nodes that are part of this instance
-    for (const runnerId of Object.keys(runnerToShard)) {
-      const nodeId = runnerToNode[runnerId];
-      if (!nodeId) continue;
-
-      const nodeDownloads = downloadsData[nodeId];
-      if (!Array.isArray(nodeDownloads)) continue;
-
-      for (const downloadWrapped of nodeDownloads) {
-        if (!downloadWrapped || typeof downloadWrapped !== "object") continue;
-
-        const keys = Object.keys(downloadWrapped as Record<string, unknown>);
-        if (keys.length !== 1) continue;
-
-        const downloadKind = keys[0];
-        const downloadPayload = (downloadWrapped as Record<string, unknown>)[
-          downloadKind
-        ] as Record<string, unknown>;
-
-        // Handle DownloadFailed - return immediately with error info
-        if (downloadKind === "DownloadFailed") {
-          const downloadModelId = extractModelIdFromDownload(downloadPayload);
-          if (
-            instanceModelId &&
-            downloadModelId &&
-            downloadModelId === instanceModelId
-          ) {
-            return {
-              isDownloading: false,
-              isFailed: true,
-              errorMessage:
-                (downloadPayload.errorMessage as string) || "Download failed",
-              progress: null,
-              statusText: "FAILED",
-              perNode: [],
-            };
-          }
-        }
-
-        if (
-          downloadKind !== "DownloadOngoing" &&
-          downloadKind !== "DownloadPending"
-        )
-          continue;
-        if (!downloadPayload) continue;
-
-        // Check if this download is for this instance's model
-        const downloadModelId = extractModelIdFromDownload(downloadPayload);
-        if (
-          instanceModelId &&
-          downloadModelId &&
-          downloadModelId === instanceModelId
-        ) {
-          // For DownloadPending with partial bytes, synthesize progress
-          let progress: DownloadProgress | null;
-          if (downloadKind === "DownloadPending") {
-            const pendingDownloaded = getBytes(
-              downloadPayload.downloaded ??
-                downloadPayload.downloaded_bytes ??
-                downloadPayload.downloadedBytes,
-            );
-            const pendingTotal = getBytes(
-              downloadPayload.total ??
-                downloadPayload.total_bytes ??
-                downloadPayload.totalBytes,
-            );
-            if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
-            isDownloading = true;
-            progress = {
-              totalBytes: pendingTotal,
-              downloadedBytes: pendingDownloaded,
-              speed: 0,
-              etaMs: 0,
-              percentage:
-                pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
-              completedFiles: 0,
-              totalFiles: 0,
-              files: [],
-            };
-          } else {
-            isDownloading = true;
-            progress = parseDownloadProgress(downloadPayload);
-          }
-
-          if (progress) {
-            // Sum all values across nodes - each node downloads independently
-            totalBytes += progress.totalBytes;
-            downloadedBytes += progress.downloadedBytes;
-            totalSpeed += progress.speed;
-            completedFiles += progress.completedFiles;
-            totalFiles += progress.totalFiles;
-            allFiles.push(...progress.files);
-
-            const nodeName =
-              data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
-            perNode.push({ nodeId, nodeName, progress });
-          }
-        }
-      }
-    }
-
-    if (!isDownloading) {
-      // Check runner status for other states
+    if (!instanceModelId) {
       const statusInfo = deriveInstanceStatus(instanceWrapped);
       return {
         isDownloading: false,
@@ -1853,26 +1775,49 @@
       };
     }
 
-    // ETA = total remaining bytes / total speed across all nodes
-    const remainingBytes = totalBytes - downloadedBytes;
-    const etaMs = totalSpeed > 0 ? (remainingBytes / totalSpeed) * 1000 : 0;
+    // Get node IDs assigned to this instance
+    const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
+    const runnerToShard = inst.shardAssignments?.runnerToShard || {};
+    const runnerToNode: Record<string, string> = {};
+    for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) {
+      runnerToNode[runnerId] = nodeId;
+    }
+    const instanceNodeIds = Object.keys(runnerToShard)
+      .map((runnerId) => runnerToNode[runnerId])
+      .filter(Boolean);
+
+    const result = collectDownloadStatus(instanceModelId, instanceNodeIds);
+
+    if (result.failedError) {
+      return {
+        isDownloading: false,
+        isFailed: true,
+        errorMessage: result.failedError,
+        progress: null,
+        statusText: "FAILED",
+        perNode: [],
+      };
+    }
+
+    if (!result.isDownloading) {
+      const statusInfo = deriveInstanceStatus(instanceWrapped);
+      return {
+        isDownloading: false,
+        isFailed: statusInfo.statusText === "FAILED",
+        errorMessage: null,
+        progress: null,
+        statusText: statusInfo.statusText,
+        perNode: result.perNode,
+      };
+    }
 
     return {
       isDownloading: true,
       isFailed: false,
       errorMessage: null,
-      progress: {
-        totalBytes,
-        downloadedBytes,
-        speed: totalSpeed,
-        etaMs,
-        percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
-        completedFiles,
-        totalFiles,
-        files: allFiles,
-      },
+      progress: result.progress,
       statusText: "DOWNLOADING",
-      perNode,
+      perNode: result.perNode,
     };
   }
 
@@ -4630,7 +4575,7 @@
                 type="button"
                 onclick={() => {
                   completeOnboarding();
-                  sendMessage(chip);
+                  sendMessage(chip, undefined, thinkingEnabled());
                 }}
                 class="px-4 py-2 rounded-full border border-white/10 bg-white/5 text-sm text-white/60 hover:bg-white/10 hover:text-white/80 hover:border-white/20 transition-all duration-200 cursor-pointer"
               >
@@ -5369,10 +5314,10 @@
                             <div
                               class="mt-2 space-y-2 max-h-48 overflow-y-auto pr-1"
                             >
-                              {#each downloadInfo.perNode as nodeProg}
+                              {#each downloadInfo.perNode.filter((n) => n.status === "downloading" && n.progress) as nodeProg}
                                 {@const nodePercent = Math.min(
                                   100,
-                                  Math.max(0, nodeProg.progress.percentage),
+                                  Math.max(0, nodeProg.percentage),
                                 )}
                                 {@const isExpanded =
                                   instanceDownloadExpandedNodes.has(
@@ -5428,15 +5373,17 @@
                                     >
                                       <span
                                         >{formatBytes(
-                                          nodeProg.progress.downloadedBytes,
+                                          nodeProg.progress?.downloadedBytes ??
+                                            0,
                                         )} / {formatBytes(
-                                          nodeProg.progress.totalBytes,
+                                          nodeProg.progress?.totalBytes ?? 0,
                                         )}</span
                                       >
                                       <span
-                                        >{formatSpeed(nodeProg.progress.speed)} •
-                                        ETA {formatEta(
-                                          nodeProg.progress.etaMs,
+                                        >{formatSpeed(
+                                          nodeProg.progress?.speed ?? 0,
+                                        )} • ETA {formatEta(
+                                          nodeProg.progress?.etaMs ?? 0,
                                         )}</span
                                       >
                                     </div>
@@ -5444,14 +5391,14 @@
 
                                   {#if isExpanded}
                                     <div class="mt-2 space-y-1.5">
-                                      {#if nodeProg.progress.files.length === 0}
+                                      {#if nodeProg.progress?.files ?? [].length === 0}
                                         <div
                                           class="text-[11px] font-mono text-exo-light-gray/70"
                                         >
                                           No file details reported.
                                         </div>
                                       {:else}
-                                        {#each nodeProg.progress.files as f}
+                                        {#each nodeProg.progress?.files ?? [] as f}
                                           {@const filePercent = Math.min(
                                             100,
                                             Math.max(0, f.percentage ?? 0),
@@ -5927,12 +5874,15 @@
                 )}
                 {@const allPreviews = filteredPreviews()}
                 {#if selectedModel && allPreviews.length > 0}
-                  {@const downloadStatus = getModelDownloadStatus(
-                    selectedModel.id,
-                  )}
                   {@const tags = modelTags()[selectedModel.id] || []}
                   <div class="space-y-3">
                     {#each allPreviews as apiPreview, i}
+                      {@const downloadStatus = getModelDownloadStatus(
+                        selectedModel.id,
+                        apiPreview.memory_delta_by_node
+                          ? Object.keys(apiPreview.memory_delta_by_node)
+                          : undefined,
+                      )}
                       <div
                         role="group"
                         onmouseenter={() => {
@@ -6120,7 +6070,7 @@
                           onclick={() => {
                             chatLaunchState = "idle";
                             selectedChatCategory = null;
-                            sendMessage(prompt);
+                            sendMessage(prompt, undefined, thinkingEnabled());
                           }}
                           class="text-left px-3 py-2.5 text-xs text-exo-light-gray hover:text-white font-mono rounded-lg border border-exo-medium-gray/30 hover:border-exo-yellow/30 bg-exo-dark-gray/30 hover:bg-exo-dark-gray/60 transition-all duration-200 cursor-pointer"
                         >
@@ -6503,10 +6453,10 @@
                               <div
                                 class="mt-2 space-y-2 max-h-48 overflow-y-auto pr-1"
                               >
-                                {#each downloadInfo.perNode as nodeProg}
+                                {#each downloadInfo.perNode.filter((n) => n.status === "downloading" && n.progress) as nodeProg}
                                   {@const nodePercent = Math.min(
                                     100,
-                                    Math.max(0, nodeProg.progress.percentage),
+                                    Math.max(0, nodeProg.percentage),
                                   )}
                                   {@const isExpanded =
                                     instanceDownloadExpandedNodes.has(
@@ -6565,16 +6515,17 @@
                                       >
                                         <span
                                           >{formatBytes(
-                                            nodeProg.progress.downloadedBytes,
+                                            nodeProg.progress
+                                              ?.downloadedBytes ?? 0,
                                           )} / {formatBytes(
-                                            nodeProg.progress.totalBytes,
+                                            nodeProg.progress?.totalBytes ?? 0,
                                           )}</span
                                         >
                                         <span
                                           >{formatSpeed(
-                                            nodeProg.progress.speed,
+                                            nodeProg.progress?.speed ?? 0,
                                           )} • ETA {formatEta(
-                                            nodeProg.progress.etaMs,
+                                            nodeProg.progress?.etaMs ?? 0,
                                           )}</span
                                         >
                                       </div>
@@ -6582,14 +6533,14 @@
 
                                     {#if isExpanded}
                                       <div class="mt-2 space-y-1.5">
-                                        {#if nodeProg.progress.files.length === 0}
+                                        {#if nodeProg.progress?.files ?? [].length === 0}
                                           <div
                                             class="text-[11px] font-mono text-exo-light-gray/70"
                                           >
                                             No file details reported.
                                           </div>
                                         {:else}
-                                          {#each nodeProg.progress.files as f}
+                                          {#each nodeProg.progress?.files ?? [] as f}
                                             {@const filePercent = Math.min(
                                               100,
                                               Math.max(0, f.percentage ?? 0),

flake.nix

@@ -72,7 +72,7 @@
       ];
 
       perSystem =
-        { config, self', inputs', pkgs, lib, system, ... }:
+        { config, self', pkgs, lib, system, ... }:
         let
           # Use pinned nixpkgs for swift-format (swift is broken on x86_64-linux in newer nixpkgs)
           pkgsSwift = import inputs.nixpkgs-swift { inherit system; };
@@ -84,6 +84,17 @@
             config.allowUnfreePredicate = pkg: (pkg.pname or "") == "metal-toolchain";
             overlays = [
               (import ./nix/apple-sdk-overlay.nix)
+              (final: prev: {
+                macmon = prev.macmon.overrideAttrs (_: {
+                  version = "git";
+                  src = final.fetchFromGitHub {
+                    owner = "swiftraccoon";
+                    repo = "macmon";
+                    rev = "9154d234f763fbeffdcb4135d0bbbaf80609699b";
+                    hash = "sha256-CwhilKNbs5XL9/tF5DMwyPBlE/hpmjGNTuxQ36sM50M=";
+                  };
+                });
+              })
             ];
           };
           treefmt = {

justfile

@@ -1,5 +1,8 @@
 export NIX_CONFIG := "extra-experimental-features = nix-command flakes"
 
+default: lint fmt
+all: lint fmt check
+
 fmt:
     treefmt || nix fmt
 
@@ -31,6 +34,10 @@ build-dashboard:
 package:
     uv run pyinstaller packaging/pyinstaller/exo.spec
 
+build-app: package
+    xcodebuild build -project app/EXO/EXO.xcodeproj -scheme EXO -configuration Debug -derivedDataPath app/EXO/build
+    @echo "\nBuild complete. Run with:\n  open {{justfile_directory()}}/app/EXO/build/Build/Products/Debug/EXO.app"
+
 clean:
     rm -rf **/__pycache__
     rm -rf target/

packaging/pyinstaller/exo.spec

@@ -71,7 +71,9 @@ MACMON_PATH = shutil.which("macmon")
 if MACMON_PATH is None:
     raise SystemExit(
         "macmon binary not found in PATH. "
-        "Install it via: brew install macmon"
+        "Install the pinned fork used by exo via: "
+        "cargo install --git https://github.com/swiftraccoon/macmon "
+        "--rev 9154d234f763fbeffdcb4135d0bbbaf80609699b macmon --force"
     )
 
 BINARIES: list[tuple[str, str]] = [
@@ -120,4 +122,3 @@ coll = COLLECT(
     upx_exclude=[],
     name="exo",
 )
-

pyproject.toml

@@ -25,7 +25,7 @@ dependencies = [
   "openai-harmony>=0.0.8",
   "httpx>=0.28.1",
   "tomlkit>=0.14.0",
-  "mflux==0.16.9",
+  "mflux==0.17.2",
   "python-multipart>=0.0.21",
   "msgspec>=0.19.0",
   "zstandard>=0.23.0",
@@ -61,7 +61,7 @@ members = ["rust/exo_pyo3_bindings", "bench"]
 [tool.uv.sources]
 exo_pyo3_bindings = { workspace = true }
 mlx = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git", branch = "address-rdma-gpu-locks", marker = "sys_platform == 'darwin'" }
-mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/eval-left-padding-in-batched-rotation" }
+mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/fix-deepseek-v32-indexer" }
 # Uncomment to use local mlx/mlx-lm development versions:
 # mlx = { path = "/Users/Shared/mlx", editable=true }
 # mlx-lm = { path = "/Users/Shared/mlx-lm", editable=true }

resources/inference_model_cards/mlx-community--DeepSeek-V3.2-4bit.toml

@@ -0,0 +1,13 @@
+model_id = "mlx-community/DeepSeek-V3.2-4bit"
+n_layers = 61
+hidden_size = 7168
+num_key_value_heads = 128
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "deepseek"
+quantization = "4bit"
+base_model = "DeepSeek V3.2"
+capabilities = ["text", "thinking", "thinking_toggle"]
+
+[storage_size]
+in_bytes = 378086226621

resources/inference_model_cards/mlx-community--DeepSeek-V3.2-8bit.toml

@@ -0,0 +1,13 @@
+model_id = "mlx-community/DeepSeek-V3.2-8bit"
+n_layers = 61
+hidden_size = 7168
+num_key_value_heads = 128
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "deepseek"
+quantization = "8bit"
+base_model = "DeepSeek V3.2"
+capabilities = ["text", "thinking", "thinking_toggle"]
+
+[storage_size]
+in_bytes = 755957120916

rust/exo_pyo3_bindings/exo_pyo3_bindings.pyi

@@ -42,7 +42,7 @@ class MessageTooLargeError(builtins.Exception):
 
 @typing.final
 class NetworkingHandle:
-    def __new__(cls, identity: Keypair) -> NetworkingHandle: ...
+    def __new__(cls, identity: Keypair, bootstrap_peers: typing.Sequence[builtins.str], listen_port: builtins.int) -> NetworkingHandle: ...
     async def gossipsub_subscribe(self, topic: builtins.str) -> builtins.bool:
         r"""
         Subscribe to a `GossipSub` topic.

rust/exo_pyo3_bindings/src/networking.rs

@@ -180,7 +180,12 @@ impl PyNetworkingHandle {
     // ---- Lifecycle management methods ----
 
     #[new]
-    fn py_new(identity: Bound<'_, PyKeypair>) -> PyResult<Self> {
+    #[pyo3(signature = (identity, bootstrap_peers, listen_port))]
+    fn py_new(
+        identity: Bound<'_, PyKeypair>,
+        bootstrap_peers: Vec<String>,
+        listen_port: u16,
+    ) -> PyResult<Self> {
         // create communication channels
         let (to_swarm, from_client) = mpsc::channel(MPSC_CHANNEL_SIZE);
 
@@ -189,7 +194,9 @@ impl PyNetworkingHandle {
 
         // create networking swarm (within tokio context!! or it crashes)
         let _guard = pyo3_async_runtimes::tokio::get_runtime().enter();
-        let swarm = create_swarm(identity, from_client).pyerr()?.into_stream();
+        let swarm = create_swarm(identity, from_client, bootstrap_peers, listen_port)
+            .pyerr()?
+            .into_stream();
 
         Ok(Self {
             swarm: Arc::new(Mutex::new(swarm)),

rust/networking/examples/chatroom.rs

@@ -16,9 +16,14 @@ async fn main() {
     let (to_swarm, from_client) = mpsc::channel(20);
 
     // Configure swarm
-    let mut swarm = swarm::create_swarm(identity::Keypair::generate_ed25519(), from_client)
-        .expect("Swarm creation failed")
-        .into_stream();
+    let mut swarm = swarm::create_swarm(
+        identity::Keypair::generate_ed25519(),
+        from_client,
+        vec![],
+        0,
+    )
+    .expect("Swarm creation failed")
+    .into_stream();
 
     // Create a Gossipsub topic & subscribe
     let (tx, rx) = oneshot::channel();

rust/networking/src/discovery.rs

@@ -104,6 +104,7 @@ pub struct Behaviour {
     // state-tracking for managed behaviors & mDNS-discovered peers
     managed: managed::Behaviour,
     mdns_discovered: HashMap<PeerId, BTreeSet<Multiaddr>>,
+    bootstrap_peers: Vec<Multiaddr>,
 
     retry_delay: Delay, // retry interval
 
@@ -112,10 +113,11 @@ pub struct Behaviour {
 }
 
 impl Behaviour {
-    pub fn new(keypair: &identity::Keypair) -> io::Result<Self> {
+    pub fn new(keypair: &identity::Keypair, bootstrap_peers: Vec<Multiaddr>) -> io::Result<Self> {
         Ok(Self {
             managed: managed::Behaviour::new(keypair)?,
             mdns_discovered: HashMap::new(),
+            bootstrap_peers,
             retry_delay: Delay::new(RETRY_CONNECT_INTERVAL),
             pending_events: WakerDeque::new(),
         })
@@ -368,6 +370,12 @@ impl NetworkBehaviour for Behaviour {
                     self.dial(p, ma)
                 }
             }
+            // dial bootstrap peers (for environments where mDNS is unavailable)
+            for addr in &self.bootstrap_peers {
+                self.pending_events.push_back(ToSwarm::Dial {
+                    opts: DialOpts::unknown_peer_id().address(addr.clone()).build(),
+                })
+            }
             self.retry_delay.reset(RETRY_CONNECT_INTERVAL) // reset timeout
         }
 

rust/networking/src/swarm.rs

@@ -142,19 +142,29 @@ fn filter_swarm_event(event: SwarmEvent<BehaviourEvent>) -> Option<FromSwarm> {
     }
 }
 
-/// Create and configure a swarm which listens to all ports on OS
+/// Create and configure a swarm.
+///
+/// - `listen_port`: TCP port to listen on. `0` lets the OS assign one.
+/// - `bootstrap_peers`: multiaddrs to dial for environments without mDNS.
 pub fn create_swarm(
     keypair: identity::Keypair,
     from_client: mpsc::Receiver<ToSwarm>,
+    bootstrap_peers: Vec<String>,
+    listen_port: u16,
 ) -> alias::AnyResult<Swarm> {
+    let parsed_bootstrap_peers: Vec<libp2p::Multiaddr> = bootstrap_peers
+        .iter()
+        .filter(|s| !s.is_empty())
+        .filter_map(|s| s.parse().ok())
+        .collect();
+
     let mut swarm = SwarmBuilder::with_existing_identity(keypair)
         .with_tokio()
         .with_other_transport(tcp_transport)?
-        .with_behaviour(Behaviour::new)?
+        .with_behaviour(|keypair| Behaviour::new(keypair, parsed_bootstrap_peers))?
         .build();
 
-    // Listen on all interfaces and whatever port the OS assigns
-    swarm.listen_on("/ip4/0.0.0.0/tcp/0".parse()?)?;
+    swarm.listen_on(format!("/ip4/0.0.0.0/tcp/{listen_port}").parse()?)?;
     Ok(Swarm { swarm, from_client })
 }
 
@@ -246,9 +256,12 @@ mod behaviour {
     }
 
     impl Behaviour {
-        pub fn new(keypair: &identity::Keypair) -> alias::AnyResult<Self> {
+        pub fn new(
+            keypair: &identity::Keypair,
+            bootstrap_peers: Vec<libp2p::Multiaddr>,
+        ) -> alias::AnyResult<Self> {
             Ok(Self {
-                discovery: discovery::Behaviour::new(keypair)?,
+                discovery: discovery::Behaviour::new(keypair, bootstrap_peers)?,
                 gossipsub: gossipsub_behaviour(keypair),
             })
         }

rust/networking/tests/bootstrap_peers.rs

@@ -0,0 +1,107 @@
+use futures_lite::StreamExt;
+use networking::swarm::{FromSwarm, create_swarm};
+use std::time::Duration;
+use tokio::sync::mpsc;
+use tokio::time::timeout;
+
+/// Helper: find a free TCP port.
+fn free_port() -> u16 {
+    let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+    listener.local_addr().unwrap().port()
+}
+
+/// Two nodes connect via bootstrap peers — no mDNS needed.
+///
+/// Node A listens on a fixed port. Node B bootstraps to A's address.
+/// We verify that B emits `FromSwarm::Discovered` for A's peer ID.
+#[tokio::test]
+async fn two_nodes_connect_via_bootstrap_peers() {
+    let port_a = free_port();
+
+    // Node A: listens on a known port, no bootstrap peers
+    let keypair_a = libp2p::identity::Keypair::generate_ed25519();
+    let peer_id_a = keypair_a.public().to_peer_id();
+    let (_tx_a, rx_a) = mpsc::channel(16);
+    let swarm_a = create_swarm(keypair_a, rx_a, vec![], port_a).expect("create swarm A");
+    let mut stream_a = swarm_a.into_stream();
+
+    // Node B: bootstraps to A's address
+    let keypair_b = libp2p::identity::Keypair::generate_ed25519();
+    let (_tx_b, rx_b) = mpsc::channel(16);
+    let swarm_b = create_swarm(
+        keypair_b,
+        rx_b,
+        vec![format!("/ip4/127.0.0.1/tcp/{port_a}")],
+        0,
+    )
+    .expect("create swarm B");
+    let mut stream_b = swarm_b.into_stream();
+
+    // Wait for B to discover A (connection established)
+    let connected = timeout(Duration::from_secs(10), async {
+        loop {
+            tokio::select! {
+                Some(event) = stream_a.next() => {
+                    // A will also see B connect, but we check from B's perspective
+                    let _ = event;
+                }
+                Some(event) = stream_b.next() => {
+                    if let FromSwarm::Discovered { peer_id } = event {
+                        if peer_id == peer_id_a {
+                            return true;
+                        }
+                    }
+                }
+            }
+        }
+    })
+    .await;
+
+    assert!(
+        connected.is_ok() && connected.unwrap(),
+        "Node B should discover Node A via bootstrap peer"
+    );
+}
+
+/// Empty bootstrap peers should work (backward compatible).
+#[tokio::test]
+async fn create_swarm_with_empty_bootstrap_peers() {
+    let keypair = libp2p::identity::Keypair::generate_ed25519();
+    let (_tx, rx) = mpsc::channel(16);
+    let swarm = create_swarm(keypair, rx, vec![], 0);
+    assert!(
+        swarm.is_ok(),
+        "create_swarm with no bootstrap peers should succeed"
+    );
+}
+
+/// Invalid multiaddr strings are silently filtered out.
+#[tokio::test]
+async fn create_swarm_ignores_invalid_bootstrap_addrs() {
+    let keypair = libp2p::identity::Keypair::generate_ed25519();
+    let (_tx, rx) = mpsc::channel(16);
+    let swarm = create_swarm(
+        keypair,
+        rx,
+        vec![
+            "not-a-valid-multiaddr".to_string(),
+            "".to_string(),
+            "/ip4/10.0.0.1/tcp/30000".to_string(), // valid
+        ],
+        0,
+    );
+    assert!(
+        swarm.is_ok(),
+        "create_swarm should succeed even with invalid bootstrap addrs"
+    );
+}
+
+/// Fixed listen port works correctly.
+#[tokio::test]
+async fn create_swarm_with_fixed_port() {
+    let port = free_port();
+    let keypair = libp2p::identity::Keypair::generate_ed25519();
+    let (_tx, rx) = mpsc::channel(16);
+    let swarm = create_swarm(keypair, rx, vec![], port);
+    assert!(swarm.is_ok(), "create_swarm with fixed port should succeed");
+}

src/exo/api/adapters/chat_completions.py

@@ -4,7 +4,7 @@ import time
 from collections.abc import AsyncGenerator
 from typing import Any
 
-from exo.shared.types.api import (
+from exo.api.types import (
     ChatCompletionChoice,
     ChatCompletionMessage,
     ChatCompletionMessageText,
@@ -202,6 +202,8 @@ async def generate_chat_stream(
                     usage=last_usage,
                 )
                 yield f"data: {tool_response.model_dump_json()}\n\n"
+                if chunk.stats is not None:
+                    yield f": generation_stats {chunk.stats.model_dump_json()}\n\n"
                 yield "data: [DONE]\n\n"
                 return
 
@@ -216,7 +218,10 @@ async def generate_chat_stream(
                 yield f"data: {chunk_response.model_dump_json()}\n\n"
 
                 if chunk.finish_reason is not None:
+                    if chunk.stats is not None:
+                        yield f": generation_stats {chunk.stats.model_dump_json()}\n\n"
                     yield "data: [DONE]\n\n"
+                    return
 
 
 async def collect_chat_response(

src/exo/api/adapters/claude.py

@@ -5,14 +5,8 @@ import re
 from collections.abc import AsyncGenerator
 from typing import Any
 
-from exo.shared.types.api import FinishReason, Usage
-from exo.shared.types.chunks import (
-    ErrorChunk,
-    PrefillProgressChunk,
-    TokenChunk,
-    ToolCallChunk,
-)
-from exo.shared.types.claude_api import (
+from exo.api.types import FinishReason, Usage
+from exo.api.types.claude_api import (
     ClaudeContentBlock,
     ClaudeContentBlockDeltaEvent,
     ClaudeContentBlockStartEvent,
@@ -35,6 +29,12 @@ from exo.shared.types.claude_api import (
     ClaudeToolUseBlock,
     ClaudeUsage,
 )
+from exo.shared.types.chunks import (
+    ErrorChunk,
+    PrefillProgressChunk,
+    TokenChunk,
+    ToolCallChunk,
+)
 from exo.shared.types.common import CommandId
 from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
 

src/exo/api/adapters/ollama.py

@@ -4,14 +4,7 @@ import json
 from collections.abc import AsyncGenerator
 from typing import Any
 
-from exo.shared.types.chunks import (
-    ErrorChunk,
-    PrefillProgressChunk,
-    TokenChunk,
-    ToolCallChunk,
-)
-from exo.shared.types.common import CommandId
-from exo.shared.types.ollama_api import (
+from exo.api.types.ollama_api import (
     OllamaChatRequest,
     OllamaChatResponse,
     OllamaDoneReason,
@@ -21,6 +14,13 @@ from exo.shared.types.ollama_api import (
     OllamaToolCall,
     OllamaToolFunction,
 )
+from exo.shared.types.chunks import (
+    ErrorChunk,
+    PrefillProgressChunk,
+    TokenChunk,
+    ToolCallChunk,
+)
+from exo.shared.types.common import CommandId
 from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
 
 

src/exo/api/adapters/responses.py

@@ -4,15 +4,8 @@ from collections.abc import AsyncGenerator
 from itertools import count
 from typing import Any
 
-from exo.shared.types.api import Usage
-from exo.shared.types.chunks import (
-    ErrorChunk,
-    PrefillProgressChunk,
-    TokenChunk,
-    ToolCallChunk,
-)
-from exo.shared.types.common import CommandId
-from exo.shared.types.openai_responses import (
+from exo.api.types import Usage
+from exo.api.types.openai_responses import (
     FunctionCallInputItem,
     ResponseCompletedEvent,
     ResponseContentPart,
@@ -42,6 +35,13 @@ from exo.shared.types.openai_responses import (
     ResponseTextDoneEvent,
     ResponseUsage,
 )
+from exo.shared.types.chunks import (
+    ErrorChunk,
+    PrefillProgressChunk,
+    TokenChunk,
+    ToolCallChunk,
+)
+from exo.shared.types.common import CommandId
 from exo.shared.types.text_generation import (
     InputMessage,
     TextGenerationTaskParams,

src/exo/api/main.py

@@ -21,17 +21,17 @@ from hypercorn.config import Config
 from hypercorn.typing import ASGIFramework
 from loguru import logger
 
-from exo.master.adapters.chat_completions import (
+from exo.api.adapters.chat_completions import (
     chat_request_to_text_generation,
     collect_chat_response,
     generate_chat_stream,
 )
-from exo.master.adapters.claude import (
+from exo.api.adapters.claude import (
     claude_request_to_text_generation,
     collect_claude_response,
     generate_claude_stream,
 )
-from exo.master.adapters.ollama import (
+from exo.api.adapters.ollama import (
     collect_ollama_chat_response,
     collect_ollama_generate_response,
     generate_ollama_chat_stream,
@@ -39,34 +39,12 @@ from exo.master.adapters.ollama import (
     ollama_generate_request_to_text_generation,
     ollama_request_to_text_generation,
 )
-from exo.master.adapters.responses import (
+from exo.api.adapters.responses import (
     collect_responses_response,
     generate_responses_stream,
     responses_request_to_text_generation,
 )
-from exo.master.event_log import DiskEventLog
-from exo.master.image_store import ImageStore
-from exo.master.placement import place_instance as get_instance_placements
-from exo.shared.apply import apply
-from exo.shared.constants import (
-    DASHBOARD_DIR,
-    EXO_CACHE_HOME,
-    EXO_EVENT_LOG_DIR,
-    EXO_IMAGE_CACHE_DIR,
-    EXO_MAX_CHUNK_SIZE,
-    EXO_TRACING_CACHE_DIR,
-)
-from exo.shared.election import ElectionMessage
-from exo.shared.logging import InterceptLogger
-from exo.shared.models.model_cards import (
-    ModelCard,
-    ModelId,
-    delete_custom_card,
-    get_model_cards,
-    is_custom_card,
-)
-from exo.shared.tracing import TraceEvent, compute_stats, export_trace, load_trace_file
-from exo.shared.types.api import (
+from exo.api.types import (
     AddCustomModelParams,
     AdvancedImageParams,
     BenchChatCompletionRequest,
@@ -114,6 +92,47 @@ from exo.shared.types.api import (
     TraceStatsResponse,
     normalize_image_size,
 )
+from exo.api.types.claude_api import (
+    ClaudeMessagesRequest,
+    ClaudeMessagesResponse,
+)
+from exo.api.types.ollama_api import (
+    OllamaChatRequest,
+    OllamaChatResponse,
+    OllamaGenerateRequest,
+    OllamaGenerateResponse,
+    OllamaModelDetails,
+    OllamaModelTag,
+    OllamaPsModel,
+    OllamaPsResponse,
+    OllamaShowRequest,
+    OllamaShowResponse,
+    OllamaTagsResponse,
+)
+from exo.api.types.openai_responses import (
+    ResponsesRequest,
+    ResponsesResponse,
+)
+from exo.master.image_store import ImageStore
+from exo.master.placement import place_instance as get_instance_placements
+from exo.shared.apply import apply
+from exo.shared.constants import (
+    DASHBOARD_DIR,
+    EXO_CACHE_HOME,
+    EXO_EVENT_LOG_DIR,
+    EXO_IMAGE_CACHE_DIR,
+    EXO_MAX_CHUNK_SIZE,
+    EXO_TRACING_CACHE_DIR,
+)
+from exo.shared.election import ElectionMessage
+from exo.shared.logging import InterceptLogger
+from exo.shared.models.model_cards import (
+    ModelCard,
+    ModelId,
+    get_card,
+    get_model_cards,
+)
+from exo.shared.tracing import TraceEvent, compute_stats, export_trace, load_trace_file
 from exo.shared.types.chunks import (
     ErrorChunk,
     ImageChunk,
@@ -122,13 +141,11 @@ from exo.shared.types.chunks import (
     TokenChunk,
     ToolCallChunk,
 )
-from exo.shared.types.claude_api import (
-    ClaudeMessagesRequest,
-    ClaudeMessagesResponse,
-)
 from exo.shared.types.commands import (
+    AddCustomModelCard,
     Command,
     CreateInstance,
+    DeleteCustomModelCard,
     DeleteDownload,
     DeleteInstance,
     DownloadCommand,
@@ -151,29 +168,13 @@ from exo.shared.types.events import (
     TracesMerged,
 )
 from exo.shared.types.memory import Memory
-from exo.shared.types.ollama_api import (
-    OllamaChatRequest,
-    OllamaChatResponse,
-    OllamaGenerateRequest,
-    OllamaGenerateResponse,
-    OllamaModelDetails,
-    OllamaModelTag,
-    OllamaPsModel,
-    OllamaPsResponse,
-    OllamaShowRequest,
-    OllamaShowResponse,
-    OllamaTagsResponse,
-)
-from exo.shared.types.openai_responses import (
-    ResponsesRequest,
-    ResponsesResponse,
-)
 from exo.shared.types.state import State
 from exo.shared.types.worker.downloads import DownloadCompleted
 from exo.shared.types.worker.instances import Instance, InstanceId, InstanceMeta
 from exo.shared.types.worker.shards import Sharding
 from exo.utils.banner import print_startup_banner
 from exo.utils.channels import Receiver, Sender, channel
+from exo.utils.disk_event_log import DiskEventLog
 from exo.utils.power_sampler import PowerSampler
 from exo.utils.task_group import TaskGroup
 
@@ -1558,7 +1559,7 @@ class API:
                     storage_size_megabytes=card.storage_size.in_mb,
                     supports_tensor=card.supports_tensor,
                     tasks=[task.value for task in card.tasks],
-                    is_custom=is_custom_card(card.model_id),
+                    is_custom=card.is_custom,
                     family=card.family,
                     quantization=card.quantization,
                     base_model=card.base_model,
@@ -1569,7 +1570,7 @@ class API:
         )
 
     async def add_custom_model(self, payload: AddCustomModelParams) -> ModelListModel:
-        """Fetch a model from HuggingFace and save as a custom model card."""
+        """Fetch a model from HuggingFace and save as a custom model card, then sync across the cluster."""
         try:
             card = await ModelCard.fetch_from_hf(payload.model_id)
         except Exception as exc:
@@ -1577,6 +1578,13 @@ class API:
                 status_code=400, detail=f"Failed to fetch model: {exc}"
             ) from exc
 
+        await self.command_sender.send(
+            ForwarderCommand(
+                origin=self._system_id,
+                command=AddCustomModelCard(model_card=card),
+            )
+        )
+
         return ModelListModel(
             id=card.model_id,
             hugging_face_id=card.model_id,
@@ -1590,10 +1598,18 @@ class API:
         )
 
     async def delete_custom_model(self, model_id: ModelId) -> JSONResponse:
-        """Delete a user-added custom model card."""
-        deleted = await delete_custom_card(model_id)
-        if not deleted:
+        """Delete a user-added custom model card and sync deletion across the cluster."""
+        card = get_card(model_id)
+        if card is None or not card.is_custom:
             raise HTTPException(status_code=404, detail="Custom model card not found")
+
+        await self.command_sender.send(
+            ForwarderCommand(
+                origin=self._system_id,
+                command=DeleteCustomModelCard(model_id=model_id),
+            )
+        )
+
         return JSONResponse(
             {"message": "Model card deleted", "model_id": str(model_id)}
         )

src/exo/api/tests/test_api_error_handling.py

@@ -4,10 +4,11 @@ from typing import Any
 from fastapi import FastAPI, HTTPException
 from fastapi.testclient import TestClient
 
+from exo.api.main import API
+
 
 def test_http_exception_handler_formats_openai_style() -> None:
     """Test that HTTPException is converted to OpenAI-style error format."""
-    from exo.master.api import API
 
     app = FastAPI()
 

src/exo/api/tests/test_cancel_command.py

@@ -5,12 +5,12 @@ from unittest.mock import AsyncMock, MagicMock
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
+from exo.api.main import API
 from exo.shared.types.common import CommandId
 
 
 def _make_api() -> Any:
     """Create a minimal API instance with cancel route and error handler."""
-    from exo.master.api import API
 
     app = FastAPI()
     api = object.__new__(API)

src/exo/api/tests/test_claude_api.py

@@ -3,11 +3,11 @@
 import pydantic
 import pytest
 
-from exo.master.adapters.claude import (
+from exo.api.adapters.claude import (
     claude_request_to_text_generation,
     finish_reason_to_claude_stop_reason,
 )
-from exo.shared.types.claude_api import (
+from exo.api.types.claude_api import (
     ClaudeMessage,
     ClaudeMessagesRequest,
     ClaudeTextBlock,

src/exo/api/tests/test_claude_tool_use.py

@@ -4,12 +4,12 @@ import json
 from collections.abc import AsyncGenerator
 from typing import Any, cast
 
-from exo.master.adapters.claude import (
+from exo.api.adapters.claude import (
     ClaudeMessagesResponse,
     collect_claude_response,
     generate_claude_stream,
 )
-from exo.shared.types.api import ToolCallItem
+from exo.api.types import ToolCallItem
 from exo.shared.types.chunks import ErrorChunk, TokenChunk, ToolCallChunk
 from exo.shared.types.common import CommandId, ModelId
 

src/exo/api/tests/test_openai_responses_api.py

@@ -7,11 +7,11 @@ The responses adapter converts it to TextGenerationTaskParams for the pipeline.
 import pydantic
 import pytest
 
-from exo.shared.types.common import ModelId
-from exo.shared.types.openai_responses import (
+from exo.api.types.openai_responses import (
     ResponseInputMessage,
     ResponsesRequest,
 )
+from exo.shared.types.common import ModelId
 
 
 class TestResponsesRequestValidation:

src/exo/api/types/__init__.py

@@ -0,0 +1,57 @@
+from .api import AddCustomModelParams as AddCustomModelParams
+from .api import AdvancedImageParams as AdvancedImageParams
+from .api import BenchChatCompletionRequest as BenchChatCompletionRequest
+from .api import BenchChatCompletionResponse as BenchChatCompletionResponse
+from .api import BenchImageGenerationResponse as BenchImageGenerationResponse
+from .api import BenchImageGenerationTaskParams as BenchImageGenerationTaskParams
+from .api import CancelCommandResponse as CancelCommandResponse
+from .api import ChatCompletionChoice as ChatCompletionChoice
+from .api import ChatCompletionMessage as ChatCompletionMessage
+from .api import ChatCompletionMessageText as ChatCompletionMessageText
+from .api import ChatCompletionRequest as ChatCompletionRequest
+from .api import ChatCompletionResponse as ChatCompletionResponse
+from .api import CompletionTokensDetails as CompletionTokensDetails
+from .api import CreateInstanceParams as CreateInstanceParams
+from .api import CreateInstanceResponse as CreateInstanceResponse
+from .api import DeleteDownloadResponse as DeleteDownloadResponse
+from .api import DeleteInstanceResponse as DeleteInstanceResponse
+from .api import DeleteTracesRequest as DeleteTracesRequest
+from .api import DeleteTracesResponse as DeleteTracesResponse
+from .api import ErrorInfo as ErrorInfo
+from .api import ErrorResponse as ErrorResponse
+from .api import FinishReason as FinishReason
+from .api import GenerationStats as GenerationStats
+from .api import HuggingFaceSearchResult as HuggingFaceSearchResult
+from .api import ImageData as ImageData
+from .api import ImageEditsTaskParams as ImageEditsTaskParams
+from .api import ImageGenerationResponse as ImageGenerationResponse
+from .api import ImageGenerationStats as ImageGenerationStats
+from .api import ImageGenerationTaskParams as ImageGenerationTaskParams
+from .api import ImageListItem as ImageListItem
+from .api import ImageListResponse as ImageListResponse
+from .api import ImageSize as ImageSize
+from .api import Logprobs as Logprobs
+from .api import LogprobsContentItem as LogprobsContentItem
+from .api import ModelList as ModelList
+from .api import ModelListModel as ModelListModel
+from .api import NodePowerStats as NodePowerStats
+from .api import PlaceInstanceParams as PlaceInstanceParams
+from .api import PlacementPreview as PlacementPreview
+from .api import PlacementPreviewResponse as PlacementPreviewResponse
+from .api import PowerUsage as PowerUsage
+from .api import PromptTokensDetails as PromptTokensDetails
+from .api import StartDownloadParams as StartDownloadParams
+from .api import StartDownloadResponse as StartDownloadResponse
+from .api import StreamingChoiceResponse as StreamingChoiceResponse
+from .api import ToolCall as ToolCall
+from .api import ToolCallItem as ToolCallItem
+from .api import TopLogprobItem as TopLogprobItem
+from .api import TraceCategoryStats as TraceCategoryStats
+from .api import TraceEventResponse as TraceEventResponse
+from .api import TraceListItem as TraceListItem
+from .api import TraceListResponse as TraceListResponse
+from .api import TraceRankStats as TraceRankStats
+from .api import TraceResponse as TraceResponse
+from .api import TraceStatsResponse as TraceStatsResponse
+from .api import Usage as Usage
+from .api import normalize_image_size as normalize_image_size

src/exo/download/download_utils.py

@@ -744,13 +744,30 @@ async def download_shard(
         logger.debug(f"Downloading {shard.model_card.model_id=} with {allow_patterns=}")
 
     all_start_time = time.time()
-    file_list = await fetch_file_list_with_cache(
-        shard.model_card.model_id,
-        revision,
-        recursive=True,
-        skip_internet=skip_internet,
-        on_connection_lost=on_connection_lost,
-    )
+    try:
+        file_list = await fetch_file_list_with_cache(
+            shard.model_card.model_id,
+            revision,
+            recursive=True,
+            skip_internet=skip_internet,
+            on_connection_lost=on_connection_lost,
+        )
+    except FileNotFoundError:
+        not_started_progress = RepoDownloadProgress(
+            repo_id=str(shard.model_card.model_id),
+            repo_revision=revision,
+            shard=shard,
+            completed_files=0,
+            total_files=0,
+            downloaded=Memory.from_bytes(0),
+            downloaded_this_session=Memory.from_bytes(0),
+            total=Memory.from_bytes(0),
+            overall_speed=0.0,
+            overall_eta=timedelta(0),
+            status="not_started",
+            file_progress={},
+        )
+        return target_dir, not_started_progress
     filtered_file_list = list(
         filter_repo_objects(
             file_list, allow_patterns=allow_patterns, key=lambda x: x.path

src/exo/main.py

@@ -11,9 +11,9 @@ from loguru import logger
 from pydantic import PositiveInt
 
 import exo.routing.topics as topics
+from exo.api.main import API
 from exo.download.coordinator import DownloadCoordinator
 from exo.download.impl_shard_downloader import exo_shard_downloader
-from exo.master.api import API  # TODO: should API be in master?
 from exo.master.main import Master
 from exo.routing.event_router import EventRouter
 from exo.routing.router import Router, get_node_id_keypair
@@ -47,7 +47,11 @@ class Node:
         keypair = get_node_id_keypair()
         node_id = NodeId(keypair.to_node_id())
         session_id = SessionId(master_node_id=node_id, election_clock=0)
-        router = Router.create(keypair)
+        router = Router.create(
+            keypair,
+            bootstrap_peers=args.bootstrap_peers,
+            listen_port=args.libp2p_port,
+        )
         await router.register_topic(topics.GLOBAL_EVENTS)
         await router.register_topic(topics.LOCAL_EVENTS)
         await router.register_topic(topics.COMMANDS)
@@ -264,12 +268,17 @@ def main():
     mp.set_start_method("spawn", force=True)
     # TODO: Refactor the current verbosity system
     logger_setup(EXO_LOG, args.verbosity)
-    logger.info("Starting EXO")
+    logger.info(f"{'=' * 40}")
+    logger.info(f"Starting EXO | pid={os.getpid()}")
+    logger.info(f"{'=' * 40}")
     logger.info(f"EXO_LIBP2P_NAMESPACE: {os.getenv('EXO_LIBP2P_NAMESPACE')}")
 
     if args.offline:
         logger.info("Running in OFFLINE mode — no internet checks, local models only")
 
+    if args.bootstrap_peers:
+        logger.info(f"Bootstrap peers: {args.bootstrap_peers}")
+
     if args.no_batch:
         os.environ["EXO_NO_BATCH"] = "1"
         logger.info("Continuous batching disabled (--no-batch)")
@@ -306,6 +315,8 @@ class Args(CamelCaseModel):
     offline: bool = os.getenv("EXO_OFFLINE", "false").lower() == "true"
     no_batch: bool = False
     fast_synch: bool | None = None  # None = auto, True = force on, False = force off
+    bootstrap_peers: list[str] = []
+    libp2p_port: int
 
     @classmethod
     def parse(cls) -> Self:
@@ -363,6 +374,22 @@ class Args(CamelCaseModel):
             action="store_true",
             help="Disable continuous batching, use sequential generation",
         )
+        parser.add_argument(
+            "--bootstrap-peers",
+            type=lambda s: [p for p in s.split(",") if p],
+            default=os.getenv("EXO_BOOTSTRAP_PEERS", "").split(",")
+            if os.getenv("EXO_BOOTSTRAP_PEERS")
+            else [],
+            dest="bootstrap_peers",
+            help="Comma-separated libp2p multiaddrs to dial on startup (env: EXO_BOOTSTRAP_PEERS)",
+        )
+        parser.add_argument(
+            "--libp2p-port",
+            type=int,
+            default=0,
+            dest="libp2p_port",
+            help="Fixed TCP port for libp2p to listen on (0 = OS-assigned).",
+        )
         fast_synch_group = parser.add_mutually_exclusive_group()
         fast_synch_group.add_argument(
             "--fast-synch",

src/exo/master/main.py

@@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone
 import anyio
 from loguru import logger
 
-from exo.master.event_log import DiskEventLog
 from exo.master.placement import (
     add_instance_to_placements,
     cancel_unnecessary_downloads,
@@ -14,7 +13,9 @@ from exo.master.placement import (
 from exo.shared.apply import apply
 from exo.shared.constants import EXO_EVENT_LOG_DIR, EXO_TRACING_ENABLED
 from exo.shared.types.commands import (
+    AddCustomModelCard,
     CreateInstance,
+    DeleteCustomModelCard,
     DeleteInstance,
     ForwarderCommand,
     ForwarderDownloadCommand,
@@ -30,6 +31,8 @@ from exo.shared.types.commands import (
 )
 from exo.shared.types.common import CommandId, NodeId, SessionId, SystemId
 from exo.shared.types.events import (
+    CustomModelCardAdded,
+    CustomModelCardDeleted,
     Event,
     GlobalForwarderEvent,
     IndexedEvent,
@@ -61,6 +64,7 @@ from exo.shared.types.tasks import (
 )
 from exo.shared.types.worker.instances import InstanceId
 from exo.utils.channels import Receiver, Sender
+from exo.utils.disk_event_log import DiskEventLog
 from exo.utils.event_buffer import MultiSourceBuffer
 from exo.utils.task_group import TaskGroup
 
@@ -294,6 +298,7 @@ class Master:
                                 self.state.instances,
                                 self.state.node_memory,
                                 self.state.node_network,
+                                download_status=self.state.downloads,
                             )
                             transition_events = get_transition_events(
                                 self.state.instances, placement, self.state.tasks
@@ -344,6 +349,14 @@ class Master:
                                     f"Finished command {command.finished_command_id} finished"
                                 )
 
+                        case AddCustomModelCard():
+                            generated_events.append(
+                                CustomModelCardAdded(model_card=command.model_card)
+                            )
+                        case DeleteCustomModelCard():
+                            generated_events.append(
+                                CustomModelCardDeleted(model_id=command.model_id)
+                            )
                         case RequestEventLog():
                             # We should just be able to send everything, since other buffers will ignore old messages
                             # rate limit to 1000 at a time

src/exo/master/placement.py

@@ -32,7 +32,10 @@ from exo.shared.types.memory import Memory
 from exo.shared.types.profiling import MemoryUsage, NodeNetworkInfo
 from exo.shared.types.tasks import Task, TaskId, TaskStatus
 from exo.shared.types.worker.downloads import (
+    DownloadCompleted,
+    DownloadFailed,
     DownloadOngoing,
+    DownloadPending,
     DownloadProgress,
 )
 from exo.shared.types.worker.instances import (
@@ -60,6 +63,45 @@ def add_instance_to_placements(
     return {**current_instances, command.instance.instance_id: command.instance}
 
 
+def _get_node_download_fraction(
+    node_id: NodeId,
+    model_id: ModelId,
+    download_status: Mapping[NodeId, Sequence[DownloadProgress]],
+) -> float:
+    """Return the download fraction (0.0–1.0) for a model on a given node."""
+    for progress in download_status.get(node_id, []):
+        if progress.shard_metadata.model_card.model_id != model_id:
+            continue
+        match progress:
+            case DownloadCompleted():
+                return 1.0
+            case DownloadOngoing():
+                total = progress.download_progress.total.in_bytes
+                return (
+                    progress.download_progress.downloaded.in_bytes / total
+                    if total > 0
+                    else 0.0
+                )
+            case DownloadPending():
+                total = progress.total.in_bytes
+                return progress.downloaded.in_bytes / total if total > 0 else 0.0
+            case DownloadFailed():
+                return 0.0
+    return 0.0
+
+
+def _cycle_download_score(
+    cycle: Cycle,
+    model_id: ModelId,
+    download_status: Mapping[NodeId, Sequence[DownloadProgress]],
+) -> float:
+    """Sum of download fractions across all nodes in a cycle."""
+    return sum(
+        _get_node_download_fraction(node_id, model_id, download_status)
+        for node_id in cycle
+    )
+
+
 def place_instance(
     command: PlaceInstance,
     topology: Topology,
@@ -67,6 +109,7 @@ def place_instance(
     node_memory: Mapping[NodeId, MemoryUsage],
     node_network: Mapping[NodeId, NodeNetworkInfo],
     required_nodes: set[NodeId] | None = None,
+    download_status: Mapping[NodeId, Sequence[DownloadProgress]] | None = None,
 ) -> dict[InstanceId, Instance]:
     cycles = topology.get_cycles()
     candidate_cycles = list(filter(lambda it: len(it) >= command.min_nodes, cycles))
@@ -130,11 +173,21 @@ def place_instance(
         if any(topology.node_is_leaf(node_id) for node_id in cycle)
     ]
 
+    resolved_download_status = download_status or {}
+    candidate_cycles = (
+        cycles_with_leaf_nodes if cycles_with_leaf_nodes != [] else smallest_cycles
+    )
+
     selected_cycle = max(
-        cycles_with_leaf_nodes if cycles_with_leaf_nodes != [] else smallest_cycles,
-        key=lambda cycle: sum(
-            (node_memory[node_id].ram_available for node_id in cycle),
-            start=Memory(),
+        candidate_cycles,
+        key=lambda cycle: (
+            _cycle_download_score(
+                cycle, command.model_card.model_id, resolved_download_status
+            ),
+            sum(
+                (node_memory[node_id].ram_available for node_id in cycle),
+                start=Memory(),
+            ),
         ),
     )
 

src/exo/master/tests/test_placement.py

@@ -25,6 +25,12 @@ from exo.shared.types.profiling import NetworkInterfaceInfo, NodeNetworkInfo
 from exo.shared.types.tasks import TaskId, TaskStatus, TextGeneration
 from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
 from exo.shared.types.topology import Connection, SocketConnection
+from exo.shared.types.worker.downloads import (
+    DownloadCompleted,
+    DownloadFailed,
+    DownloadOngoing,
+    DownloadProgressData,
+)
 from exo.shared.types.worker.instances import (
     Instance,
     InstanceId,
@@ -33,7 +39,7 @@ from exo.shared.types.worker.instances import (
     MlxRingInstance,
 )
 from exo.shared.types.worker.runners import ShardAssignments
-from exo.shared.types.worker.shards import Sharding
+from exo.shared.types.worker.shards import PipelineShardMetadata, Sharding
 
 
 @pytest.fixture
@@ -576,3 +582,183 @@ def test_get_transition_events_delete_instance_cancels_only_matching_tasks(
     assert cancel_events[0].task_status == TaskStatus.Cancelled
     assert len(delete_events) == 1
     assert delete_events[0].instance_id == instance_id_a
+
+
+def _make_shard_metadata(model_card: ModelCard) -> PipelineShardMetadata:
+    return PipelineShardMetadata(
+        model_card=model_card,
+        device_rank=0,
+        world_size=1,
+        start_layer=0,
+        end_layer=model_card.n_layers,
+        n_layers=model_card.n_layers,
+    )
+
+
+def test_placement_prefers_cycle_with_downloaded_model(
+    model_card: ModelCard,
+) -> None:
+    """When two cycles are otherwise equal, prefer the one with the model already downloaded."""
+    topology = Topology()
+
+    model_card.storage_size = Memory.from_bytes(500)
+
+    node_a = NodeId()
+    node_b = NodeId()
+
+    node_memory = {
+        node_a: create_node_memory(1000),
+        node_b: create_node_memory(1000),
+    }
+    node_network = {
+        node_a: create_node_network(),
+        node_b: create_node_network(),
+    }
+
+    topology.add_node(node_a)
+    topology.add_node(node_b)
+    # No connections between them — two single-node cycles
+
+    shard_meta = _make_shard_metadata(model_card)
+
+    # node_b has the model fully downloaded, node_a does not
+    download_status = {
+        node_b: [
+            DownloadCompleted(
+                node_id=node_b,
+                shard_metadata=shard_meta,
+                total=model_card.storage_size,
+            ),
+        ],
+    }
+
+    cic = place_instance_command(model_card)
+    placements = place_instance(
+        cic, topology, {}, node_memory, node_network, download_status=download_status
+    )
+
+    assert len(placements) == 1
+    instance = list(placements.values())[0]
+    assigned_nodes = set(instance.shard_assignments.node_to_runner.keys())
+    assert assigned_nodes == {node_b}
+
+
+def test_placement_prefers_cycle_with_higher_download_progress(
+    model_card: ModelCard,
+) -> None:
+    """When two cycles are otherwise equal, prefer the one with more download progress."""
+    topology = Topology()
+
+    model_card.storage_size = Memory.from_bytes(1000)
+
+    node_a = NodeId()
+    node_b = NodeId()
+
+    node_memory = {
+        node_a: create_node_memory(1000),
+        node_b: create_node_memory(1000),
+    }
+    node_network = {
+        node_a: create_node_network(),
+        node_b: create_node_network(),
+    }
+
+    topology.add_node(node_a)
+    topology.add_node(node_b)
+
+    shard_meta = _make_shard_metadata(model_card)
+
+    # node_a: 30% downloaded, node_b: 80% downloaded
+    download_status = {
+        node_a: [
+            DownloadOngoing(
+                node_id=node_a,
+                shard_metadata=shard_meta,
+                download_progress=DownloadProgressData(
+                    total=Memory.from_bytes(1000),
+                    downloaded=Memory.from_bytes(300),
+                    downloaded_this_session=Memory.from_bytes(300),
+                    completed_files=0,
+                    total_files=1,
+                    speed=0.0,
+                    eta_ms=0,
+                    files={},
+                ),
+            ),
+        ],
+        node_b: [
+            DownloadOngoing(
+                node_id=node_b,
+                shard_metadata=shard_meta,
+                download_progress=DownloadProgressData(
+                    total=Memory.from_bytes(1000),
+                    downloaded=Memory.from_bytes(800),
+                    downloaded_this_session=Memory.from_bytes(800),
+                    completed_files=0,
+                    total_files=1,
+                    speed=0.0,
+                    eta_ms=0,
+                    files={},
+                ),
+            ),
+        ],
+    }
+
+    cic = place_instance_command(model_card)
+    placements = place_instance(
+        cic, topology, {}, node_memory, node_network, download_status=download_status
+    )
+
+    assert len(placements) == 1
+    instance = list(placements.values())[0]
+    assigned_nodes = set(instance.shard_assignments.node_to_runner.keys())
+    assert assigned_nodes == {node_b}
+
+
+def test_placement_does_not_prefer_cycle_with_failed_download(
+    model_card: ModelCard,
+) -> None:
+    """A failed download should count as 0% — not preferred over a node with no download history."""
+    topology = Topology()
+
+    model_card.storage_size = Memory.from_bytes(500)
+
+    node_a = NodeId()
+    node_b = NodeId()
+
+    # node_a has slightly more RAM so it would win on the RAM tiebreaker
+    node_memory = {
+        node_a: create_node_memory(1001),
+        node_b: create_node_memory(1000),
+    }
+    node_network = {
+        node_a: create_node_network(),
+        node_b: create_node_network(),
+    }
+
+    topology.add_node(node_a)
+    topology.add_node(node_b)
+
+    shard_meta = _make_shard_metadata(model_card)
+
+    # node_b has a failed download — should not be preferred
+    download_status = {
+        node_b: [
+            DownloadFailed(
+                node_id=node_b,
+                shard_metadata=shard_meta,
+                error_message="connection reset",
+            ),
+        ],
+    }
+
+    cic = place_instance_command(model_card)
+    placements = place_instance(
+        cic, topology, {}, node_memory, node_network, download_status=download_status
+    )
+
+    assert len(placements) == 1
+    instance = list(placements.values())[0]
+    assigned_nodes = set(instance.shard_assignments.node_to_runner.keys())
+    # node_a should win on RAM tiebreaker since failed download scores 0.0
+    assert assigned_nodes == {node_a}

src/exo/routing/router.py

@@ -1,3 +1,4 @@
+from collections.abc import Sequence
 from copy import copy
 from itertools import count
 from math import inf
@@ -102,8 +103,15 @@ class TopicRouter[T: CamelCaseModel]:
 
 class Router:
     @classmethod
-    def create(cls, identity: Keypair) -> "Router":
-        return cls(handle=NetworkingHandle(identity))
+    def create(
+        cls,
+        identity: Keypair,
+        bootstrap_peers: Sequence[str] = (),
+        listen_port: int = 0,
+    ) -> "Router":
+        return cls(
+            handle=NetworkingHandle(identity, list(bootstrap_peers), listen_port)
+        )
 
     def __init__(self, handle: NetworkingHandle):
         self.topic_routers: dict[str, TopicRouter[CamelCaseModel]] = {}

src/exo/shared/apply.py

@@ -7,6 +7,8 @@ from loguru import logger
 from exo.shared.types.common import NodeId
 from exo.shared.types.events import (
     ChunkGenerated,
+    CustomModelCardAdded,
+    CustomModelCardDeleted,
     Event,
     IndexedEvent,
     InputChunkReceived,
@@ -65,6 +67,8 @@ def event_apply(event: Event, state: State) -> State:
             | InputChunkReceived()
             | TracesCollected()
             | TracesMerged()
+            | CustomModelCardAdded()
+            | CustomModelCardDeleted()
         ):  # Pass-through events that don't modify state
             return state
         case InstanceCreated():
@@ -115,7 +119,13 @@ def apply_node_download_progress(event: NodeDownloadProgress, state: State) -> S
 
     replaced = False
     for i, existing_dp in enumerate(current):
-        if existing_dp.shard_metadata == dp.shard_metadata:
+        # TODO(ciaran): deduplicate by model_id for now. Will need to use
+        # shard_metadata again when pipeline and tensor downloads differ.
+        # For now this is fine
+        if (
+            existing_dp.shard_metadata.model_card.model_id
+            == dp.shard_metadata.model_card.model_id
+        ):
             current[i] = dp
             replaced = True
             break

src/exo/shared/logging.py

@@ -66,7 +66,7 @@ def logger_setup(log_file: Path | None, verbosity: int = 0):
     else:
         logger.add(
             sys.__stderr__,  # type: ignore
-            format="[ {time:HH:mm:ss.SSS} | <level>{level: <8}</level> | {name}:{function}:{line} ] <level>{message}</level>",
+            format="[ {time:YYYY-MM-DD HH:mm:ss.SSS} | <level>{level: <8}</level> | {name}:{function}:{line} ] <level>{message}</level>",
             level="DEBUG",
             colorize=True,
             enqueue=True,
@@ -76,7 +76,7 @@ def logger_setup(log_file: Path | None, verbosity: int = 0):
         logger.add(
             log_file,
             format="[ {time:YYYY-MM-DD HH:mm:ss.SSS} | {level: <8} | {name}:{function}:{line} ] {message}",
-            level="INFO",
+            level="DEBUG" if verbosity > 0 else "INFO",
             colorize=False,
             enqueue=True,
             rotation=lambda _, __: next(rotate_once),

src/exo/shared/models/model_cards.py

@@ -30,30 +30,42 @@ from exo.utils.pydantic_ext import CamelCaseModel
 # kinda ugly...
 # TODO: load search path from config.toml
 _custom_cards_dir = Path(str(EXO_CUSTOM_MODEL_CARDS_DIR))
-CARD_SEARCH_PATH = [
+_BUILTIN_CARD_DIRS = [
     Path(RESOURCES_DIR) / "inference_model_cards",
     Path(RESOURCES_DIR) / "image_model_cards",
-    _custom_cards_dir,
 ]
 
 _card_cache: dict[ModelId, "ModelCard"] = {}
 
 
-async def _refresh_card_cache():
-    for path in CARD_SEARCH_PATH:
-        async for toml_file in path.rglob("*.toml"):
-            try:
-                card = await ModelCard.load_from_path(toml_file)
-                if card.model_id not in _card_cache:
-                    _card_cache[card.model_id] = card
-            except (ValidationError, TOMLKitError):
-                pass
+async def _load_cards_from_dir(directory: Path, *, is_custom: bool) -> None:
+    """Load all TOML model cards from a directory into the cache."""
+    async for toml_file in directory.rglob("*.toml"):
+        try:
+            card = await ModelCard.load_from_path(toml_file)
+            if is_custom:
+                card = card.model_copy(update={"is_custom": True})
+            if card.model_id not in _card_cache:
+                _card_cache[card.model_id] = card
+        except (ValidationError, TOMLKitError):
+            pass
+
+
+async def _refresh_card_cache() -> None:
+    for path in _BUILTIN_CARD_DIRS:
+        await _load_cards_from_dir(path, is_custom=False)
+    await _load_cards_from_dir(_custom_cards_dir, is_custom=True)
 
 
 def _is_image_card(card: "ModelCard") -> bool:
     return any(t in (ModelTask.TextToImage, ModelTask.ImageToImage) for t in card.tasks)
 
 
+def get_card(model_id: ModelId) -> "ModelCard | None":
+    """Look up a single model card from the cache by ID."""
+    return _card_cache.get(model_id)
+
+
 async def get_model_cards() -> list["ModelCard"]:
     if len(_card_cache) == 0:
         await _refresh_card_cache()
@@ -92,6 +104,7 @@ class ModelCard(CamelCaseModel):
     capabilities: list[str] = []
     uses_cfg: bool = False
     trust_remote_code: bool = True
+    is_custom: bool = False
 
     @field_validator("tasks", mode="before")
     @classmethod
@@ -100,7 +113,7 @@ class ModelCard(CamelCaseModel):
 
     async def save(self, path: Path) -> None:
         async with await open_file(path, "w") as f:
-            py = self.model_dump(exclude_none=True)
+            py = self.model_dump(exclude_none=True, exclude={"is_custom"})
             data = tomlkit.dumps(py)  # pyright: ignore[reportUnknownMemberType]
             await f.write(data)
 
@@ -122,17 +135,24 @@ class ModelCard(CamelCaseModel):
         if (mc := _card_cache.get(model_id)) is not None:
             return mc
 
-        return await ModelCard.fetch_from_hf(model_id)
+        mc = await ModelCard.fetch_from_hf(model_id)
+        await mc.save_to_custom_dir()
+        _card_cache[model_id] = mc
+        return mc
 
     @staticmethod
     async def fetch_from_hf(model_id: ModelId) -> "ModelCard":
-        """Fetches storage size and number of layers for a Hugging Face model, returns Pydantic ModelMeta."""
+        """Fetches storage size and number of layers for a Hugging Face model, returns Pydantic ModelMeta.
+
+        This is a pure fetch — it does NOT save to disk or update the cache.
+        Persistence is handled by the event-sourcing layer (worker event handler).
+        """
         # TODO: failure if files do not exist
         config_data = await fetch_config_data(model_id)
         num_layers = config_data.layer_count
         mem_size_bytes = await fetch_safetensors_size(model_id)
 
-        mc = ModelCard(
+        return ModelCard(
             model_id=ModelId(model_id),
             storage_size=mem_size_bytes,
             n_layers=num_layers,
@@ -141,10 +161,13 @@ class ModelCard(CamelCaseModel):
             num_key_value_heads=config_data.num_key_value_heads,
             tasks=[ModelTask.TextGeneration],
             trust_remote_code=False,
+            is_custom=True,
         )
-        await mc.save_to_custom_dir()
-        _card_cache[model_id] = mc
-        return mc
+
+
+def add_to_card_cache(card: "ModelCard") -> None:
+    """Add or update a model card in the in-memory cache."""
+    _card_cache[card.model_id] = card
 
 
 async def delete_custom_card(model_id: ModelId) -> bool:
@@ -157,16 +180,6 @@ async def delete_custom_card(model_id: ModelId) -> bool:
     return False
 
 
-def is_custom_card(model_id: ModelId) -> bool:
-    """Check if a model card exists in the custom cards directory."""
-    import os
-
-    card_path = Path(str(EXO_CUSTOM_MODEL_CARDS_DIR)) / (
-        ModelId(model_id).normalize() + ".toml"
-    )
-    return os.path.isfile(str(card_path))
-
-
 class ConfigData(BaseModel):
     model_config = {"extra": "ignore"}  # Allow unknown fields
 

src/exo/shared/types/chunks.py

@@ -1,18 +1,18 @@
 from collections.abc import Generator
 from typing import Any, Literal
 
-from exo.shared.models.model_cards import ModelId
-from exo.shared.types.api import (
+from exo.api.types import (
+    FinishReason,
     GenerationStats,
     ImageGenerationStats,
+    ToolCallItem,
     TopLogprobItem,
     Usage,
 )
+from exo.shared.models.model_cards import ModelId
 from exo.utils.pydantic_ext import TaggedModel
 
-from .api import FinishReason
 from .common import CommandId
-from .worker.runner_response import ToolCallItem
 
 
 class BaseChunk(TaggedModel):

src/exo/shared/types/commands.py

@@ -1,10 +1,10 @@
 from pydantic import Field
 
-from exo.shared.models.model_cards import ModelCard, ModelId
-from exo.shared.types.api import (
+from exo.api.types import (
     ImageEditsTaskParams,
     ImageGenerationTaskParams,
 )
+from exo.shared.models.model_cards import ModelCard, ModelId
 from exo.shared.types.chunks import InputImageChunk
 from exo.shared.types.common import CommandId, NodeId, SystemId
 from exo.shared.types.text_generation import TextGenerationTaskParams
@@ -81,6 +81,14 @@ class CancelDownload(BaseCommand):
     model_id: ModelId
 
 
+class AddCustomModelCard(BaseCommand):
+    model_card: ModelCard
+
+
+class DeleteCustomModelCard(BaseCommand):
+    model_id: ModelId
+
+
 DownloadCommand = StartDownload | DeleteDownload | CancelDownload
 
 
@@ -96,6 +104,8 @@ Command = (
     | TaskCancelled
     | TaskFinished
     | SendInputChunk
+    | AddCustomModelCard
+    | DeleteCustomModelCard
 )
 
 

src/exo/shared/types/events.py

@@ -3,9 +3,10 @@ from typing import final
 
 from pydantic import Field
 
+from exo.shared.models.model_cards import ModelCard
 from exo.shared.topology import Connection
 from exo.shared.types.chunks import GenerationChunk, InputImageChunk
-from exo.shared.types.common import CommandId, Id, NodeId, SessionId, SystemId
+from exo.shared.types.common import CommandId, Id, ModelId, NodeId, SessionId, SystemId
 from exo.shared.types.tasks import Task, TaskId, TaskStatus
 from exo.shared.types.worker.downloads import DownloadProgress
 from exo.shared.types.worker.instances import Instance, InstanceId
@@ -106,6 +107,14 @@ class TopologyEdgeDeleted(BaseEvent):
     conn: Connection
 
 
+class CustomModelCardAdded(BaseEvent):
+    model_card: ModelCard
+
+
+class CustomModelCardDeleted(BaseEvent):
+    model_id: ModelId
+
+
 @final
 class TraceEventData(FrozenModel):
     name: str
@@ -147,6 +156,8 @@ Event = (
     | TopologyEdgeDeleted
     | TracesCollected
     | TracesMerged
+    | CustomModelCardAdded
+    | CustomModelCardDeleted
 )
 
 

src/exo/shared/types/tasks.py

@@ -2,7 +2,7 @@ from enum import Enum
 
 from pydantic import Field
 
-from exo.shared.types.api import (
+from exo.api.types import (
     ImageEditsTaskParams,
     ImageGenerationTaskParams,
 )

src/exo/shared/types/worker/runner_response.py

@@ -1,7 +1,7 @@
 from collections.abc import Generator
 from typing import Any, Literal
 
-from exo.shared.types.api import (
+from exo.api.types import (
     FinishReason,
     GenerationStats,
     ImageGenerationStats,

src/exo/shared/types/worker/runners.py

@@ -63,7 +63,7 @@ class RunnerShutdown(BaseRunnerStatus):
 
 
 class RunnerFailed(BaseRunnerStatus):
-    error_message: str | None = None
+    error_message: str
 
 
 RunnerStatus = (

src/exo/utils/info_gatherer/info_gatherer.py

@@ -10,7 +10,6 @@ from typing import Self, cast
 import anyio
 from anyio import fail_after, open_process, to_thread
 from anyio.streams.buffered import BufferedByteReceiveStream
-from anyio.streams.text import TextReceiveStream
 from loguru import logger
 from pydantic import ValidationError
 
@@ -288,7 +287,7 @@ class ThunderboltBridgeInfo(TaggedModel):
                 )
             )
         except Exception as e:
-            logger.warning(f"Failed to gather Thunderbolt Bridge info: {e}")
+            logger.opt(exception=e).warning("Failed to gather Thunderbolt Bridge info")
             return None
 
 
@@ -382,18 +381,61 @@ class InfoGatherer:
     rdma_ctl_poll_interval: float | None = 10 if IS_DARWIN else None
     disk_poll_interval: float | None = 30
     _tg: TaskGroup = field(init=False, default_factory=TaskGroup)
+    _psutil_memory_fallback_enabled: bool = field(init=False, default=False)
+
+    def _get_macmon_path(self) -> str | None:
+        return os.getenv("EXO_MACMON_PATH") or shutil.which("macmon")
+
+    async def _can_read_macmon_metrics(self, macmon_path: str) -> bool:
+        try:
+            with fail_after(5):
+                proc = await anyio.run_process(
+                    [macmon_path, "pipe", "--samples", "1", "--interval", "100"],
+                    check=False,
+                )
+        except Exception as e:
+            logger.opt(exception=e).warning(
+                f"Failed to validate macmon at {macmon_path}"
+            )
+            return False
+
+        if proc.returncode != 0:
+            stderr = proc.stderr.decode("utf-8", errors="replace").strip()
+            logger.warning(
+                f"macmon preflight failed with return code {proc.returncode}: "
+                f"{stderr or 'no stderr'}"
+            )
+            return False
+
+        stdout = proc.stdout.decode("utf-8", errors="replace").strip()
+        if not stdout:
+            logger.warning("macmon preflight returned no metrics")
+            return False
+
+        try:
+            MacmonMetrics.from_raw_json(stdout.splitlines()[0])
+        except ValidationError as e:
+            logger.opt(exception=e).warning(
+                "macmon preflight returned unexpected metrics JSON"
+            )
+            return False
+
+        return True
 
     async def run(self):
         async with self._tg as tg:
             if IS_DARWIN:
-                if (macmon_path := shutil.which("macmon")) is not None:
-                    tg.start_soon(self._monitor_macmon, macmon_path)
+                if (macmon_path := self._get_macmon_path()) is not None:
+                    if await self._can_read_macmon_metrics(macmon_path):
+                        tg.start_soon(self._monitor_macmon, macmon_path)
+                    else:
+                        logger.warning(
+                            f"macmon at {macmon_path} is unusable, falling back to psutil memory monitoring"
+                        )
                 else:
-                    # macmon not installed — fall back to psutil for memory
                     logger.warning(
                         "macmon not found, falling back to psutil for memory monitoring"
                     )
-                    self.memory_poll_rate = 1
                 tg.start_soon(self._monitor_system_profiler_thunderbolt_data)
                 tg.start_soon(self._monitor_thunderbolt_bridge_status)
                 tg.start_soon(self._monitor_rdma_ctl_status)
@@ -418,7 +460,7 @@ class InfoGatherer:
                 with fail_after(30):
                     await self.info_sender.send(await StaticNodeInformation.gather())
             except Exception as e:
-                logger.warning(f"Error gathering static node info: {e}")
+                logger.opt(exception=e).warning("Error gathering static node info")
             await anyio.sleep(self.static_info_poll_interval)
 
     async def _monitor_misc(self):
@@ -429,7 +471,7 @@ class InfoGatherer:
                 with fail_after(10):
                     await self.info_sender.send(await MiscData.gather())
             except Exception as e:
-                logger.warning(f"Error gathering misc data: {e}")
+                logger.opt(exception=e).warning("Error gathering misc data")
             await anyio.sleep(self.misc_poll_interval)
 
     async def _monitor_system_profiler_thunderbolt_data(self):
@@ -456,7 +498,7 @@ class InfoGatherer:
                     conns = [it for i in data if (it := i.conn()) is not None]
                     await self.info_sender.send(MacThunderboltConnections(conns=conns))
             except Exception as e:
-                logger.warning(f"Error gathering Thunderbolt data: {e}")
+                logger.opt(exception=e).warning("Error gathering Thunderbolt data")
             await anyio.sleep(self.system_profiler_interval)
 
     async def _monitor_memory_usage(self):
@@ -474,7 +516,7 @@ class InfoGatherer:
                     MemoryUsage.from_psutil(override_memory=override_memory)
                 )
             except Exception as e:
-                logger.warning(f"Error gathering memory usage: {e}")
+                logger.opt(exception=e).warning("Error gathering memory usage")
             await anyio.sleep(self.memory_poll_rate)
 
     async def _watch_system_info(self):
@@ -486,7 +528,7 @@ class InfoGatherer:
                     nics = await get_network_interfaces()
                     await self.info_sender.send(NodeNetworkInterfaces(ifaces=nics))
             except Exception as e:
-                logger.warning(f"Error gathering network interfaces: {e}")
+                logger.opt(exception=e).warning("Error gathering network interfaces")
             await anyio.sleep(self.interface_watcher_interval)
 
     async def _monitor_thunderbolt_bridge_status(self):
@@ -499,7 +541,9 @@ class InfoGatherer:
                     if curr is not None:
                         await self.info_sender.send(curr)
             except Exception as e:
-                logger.warning(f"Error gathering Thunderbolt Bridge status: {e}")
+                logger.opt(exception=e).warning(
+                    "Error gathering Thunderbolt Bridge status"
+                )
             await anyio.sleep(self.thunderbolt_bridge_poll_interval)
 
     async def _monitor_rdma_ctl_status(self):
@@ -511,7 +555,7 @@ class InfoGatherer:
                 if curr is not None:
                     await self.info_sender.send(curr)
             except Exception as e:
-                logger.warning(f"Error gathering RDMA ctl status: {e}")
+                logger.opt(exception=e).warning("Error gathering RDMA ctl status")
             await anyio.sleep(self.rdma_ctl_poll_interval)
 
     async def _monitor_disk_usage(self):
@@ -522,7 +566,7 @@ class InfoGatherer:
                 with fail_after(5):
                     await self.info_sender.send(await NodeDiskUsage.gather())
             except Exception as e:
-                logger.warning(f"Error gathering disk usage: {e}")
+                logger.opt(exception=e).warning("Error gathering disk usage")
             await anyio.sleep(self.disk_poll_interval)
 
     async def _monitor_macmon(self, macmon_path: str):
@@ -545,15 +589,21 @@ class InfoGatherer:
                     if not p.stdout:
                         logger.critical("MacMon closed stdout")
                         return
-                    stream = TextReceiveStream(BufferedByteReceiveStream(p.stdout))
+                    stream = BufferedByteReceiveStream(p.stdout)
                     while True:
                         with fail_after(read_timeout):
-                            text = await stream.receive()
-                        await self.info_sender.send(MacmonMetrics.from_raw_json(text))
+                            data = await stream.receive_until(
+                                delimiter=b"\n", max_bytes=8 * 1024
+                            )
+                            text = data.decode("utf-8", errors="replace").strip()
+                            metrics = MacmonMetrics.from_raw_json(text)
+                        await self.info_sender.send(metrics)
             except TimeoutError:
                 logger.warning(
                     f"MacMon produced no output for {read_timeout}s, restarting"
                 )
+                self.memory_poll_rate = 1
+                self._tg.start_soon(self._monitor_memory_usage)
             except CalledProcessError as e:
                 stderr_msg = "no stderr"
                 stderr_output = cast(bytes | str | None, e.stderr)
@@ -566,6 +616,10 @@ class InfoGatherer:
                 logger.warning(
                     f"MacMon failed with return code {e.returncode}: {stderr_msg}"
                 )
+                self.memory_poll_rate = 1
+                self._tg.start_soon(self._monitor_memory_usage)
             except Exception as e:
-                logger.warning(f"Error in macmon monitor: {e}")
+                logger.opt(exception=e).warning("Error in macmon monitor")
+                self.memory_poll_rate = 1
+                self._tg.start_soon(self._monitor_memory_usage)
             await anyio.sleep(self.macmon_interval)

src/exo/utils/power_sampler.py

@@ -5,7 +5,7 @@ from typing import final
 
 import anyio
 
-from exo.shared.types.api import NodePowerStats, PowerUsage
+from exo.api.types import NodePowerStats, PowerUsage
 from exo.shared.types.common import NodeId
 from exo.shared.types.profiling import SystemPerformanceProfile
 

src/exo/utils/tests/test_event_log.py

@@ -2,8 +2,8 @@ from pathlib import Path
 
 import pytest
 
-from exo.master.event_log import DiskEventLog
 from exo.shared.types.events import TestEvent
+from exo.utils.disk_event_log import DiskEventLog
 
 
 @pytest.fixture

src/exo/utils/tests/test_power_sampler.py

@@ -3,7 +3,7 @@ from collections.abc import Mapping
 import anyio
 import pytest
 
-from exo.shared.types.api import PowerUsage
+from exo.api.types import PowerUsage
 from exo.shared.types.common import NodeId
 from exo.shared.types.profiling import SystemPerformanceProfile
 from exo.utils.power_sampler import PowerSampler

src/exo/worker/engines/image/distributed_model.py

@@ -6,8 +6,8 @@ import mlx.core as mx
 from mflux.models.common.config.config import Config
 from PIL import Image
 
+from exo.api.types import AdvancedImageParams
 from exo.download.download_utils import build_model_path
-from exo.shared.types.api import AdvancedImageParams
 from exo.shared.types.worker.instances import BoundInstance
 from exo.shared.types.worker.shards import CfgShardMetadata, PipelineShardMetadata
 from exo.worker.engines.image.config import ImageModelConfig

src/exo/worker/engines/image/generate.py

@@ -9,7 +9,7 @@ from typing import Generator, Literal
 import mlx.core as mx
 from PIL import Image
 
-from exo.shared.types.api import (
+from exo.api.types import (
     AdvancedImageParams,
     ImageEditsTaskParams,
     ImageGenerationStats,

src/exo/worker/engines/image/models/qwen/wrappers.py

@@ -56,10 +56,10 @@ class QwenJointBlockWrapper(JointBlockWrapper[QwenTransformerBlock]):
         attn = self.block.attn
 
         img_mod_params = self.block.img_mod_linear(
-            self.block.img_mod_silu(text_embeddings)  # pyright: ignore[reportUnknownArgumentType]
+            self.block.img_mod_silu(text_embeddings)
         )
         txt_mod_params = self.block.txt_mod_linear(
-            self.block.txt_mod_silu(text_embeddings)  # pyright: ignore[reportUnknownArgumentType]
+            self.block.txt_mod_silu(text_embeddings)
         )
 
         img_mod1, img_mod2 = mx.split(img_mod_params, 2, axis=-1)

src/exo/worker/engines/mlx/auto_parallel.py

@@ -57,8 +57,8 @@ from mlx_lm.models.step3p5 import Model as Step35Model
 from mlx_lm.models.step3p5 import Step3p5MLP as Step35MLP
 from mlx_lm.models.step3p5 import Step3p5Model as Step35InnerModel
 
-from exo.shared.logging import logger
 from exo.shared.types.worker.shards import PipelineShardMetadata
+from exo.worker.runner.bootstrap import logger
 
 if TYPE_CHECKING:
     from mlx_lm.models.cache import Cache
@@ -480,7 +480,7 @@ def patch_tensor_model[T](model: T) -> T:
             last = cache[-1]  # pyright: ignore[reportAny]
             dep_cache = last[0] if hasattr(last, "caches") else last  # pyright: ignore[reportAny]
             if hasattr(dep_cache, "keys"):  # type: ignore
-                dep_cache.keys = mx.depends(dep_cache.keys, logits)  # pyright: ignore[reportAny,reportUnknownMemberType]
+                dep_cache.keys = mx.depends(dep_cache.keys, logits)  # pyright: ignore[reportAny]
 
         return logits
 

src/exo/worker/engines/mlx/dsml_encoding.py

@@ -4,7 +4,7 @@ from typing import Any
 
 from mlx_lm.chat_templates import deepseek_v32
 
-from exo.shared.types.api import ToolCallItem
+from exo.api.types import ToolCallItem
 
 BOS_TOKEN: str = deepseek_v32.bos_token
 EOS_TOKEN: str = deepseek_v32.eos_token
@@ -15,7 +15,28 @@ USER_TOKEN = "<\uff5cUser\uff5c>"
 ASSISTANT_TOKEN = "<\uff5cAssistant\uff5c>"
 TOOL_CALLS_START = f"<{DSML_TOKEN}function_calls>"
 TOOL_CALLS_END = f"</{DSML_TOKEN}function_calls>"
-encode_messages = deepseek_v32.encode_messages
+_ORPHAN_THINK_END = ASSISTANT_TOKEN + THINKING_END
+_FIXED_THINK_BLOCK = ASSISTANT_TOKEN + THINKING_START + "\n" + THINKING_END
+
+
+def encode_messages(
+    messages: list[dict[str, Any]],
+    thinking_mode: str = "thinking",
+    context: list[dict[str, Any]] | None = None,
+    drop_thinking: bool = True,
+    add_default_bos_token: bool = True,
+    tools: Any = None,  # pyright: ignore[reportAny]
+) -> str:
+    prompt: str = deepseek_v32.encode_messages(
+        messages,
+        thinking_mode=thinking_mode,
+        context=context,
+        drop_thinking=drop_thinking,
+        add_default_bos_token=add_default_bos_token,
+        tools=tools,
+    )
+    return prompt.replace(_ORPHAN_THINK_END, _FIXED_THINK_BLOCK)
+
 
 _INVOKE_PATTERN = re.compile(
     rf"<{re.escape(DSML_TOKEN)}invoke\s+name=\"([^\"]+)\">"

src/exo/worker/engines/mlx/generator/batch_generate.py

@@ -6,11 +6,14 @@ import mlx.core as mx
 from mlx_lm.generate import (
     BatchGenerator as MlxBatchGenerator,
 )
+from mlx_lm.generate import (
+    generation_stream,
+)
 from mlx_lm.models.cache import RotatingKVCache
 from mlx_lm.sample_utils import make_logits_processors, make_sampler
 from mlx_lm.tokenizer_utils import StreamingDetokenizer, TokenizerWrapper
 
-from exo.shared.types.api import (
+from exo.api.types import (
     CompletionTokensDetails,
     FinishReason,
     GenerationStats,
@@ -63,6 +66,7 @@ class _EngineTask:
     potential_stop_sequence_text: str = ""
     completion_tokens: int = 0
     generation_start_time: float = 0.0
+    generation_time_at_start: float = 0.0
     in_thinking: bool = False
     reasoning_tokens: int = 0
     prefill_tps: float = 0.0
@@ -75,22 +79,23 @@ class ExoBatchGenerator:
     group: mx.distributed.Group | None
     kv_prefix_cache: KVPrefixCache | None
 
-    _exo_gen: MlxBatchGenerator = field(init=False)
+    _mlx_gen: MlxBatchGenerator = field(init=False)
     _active_tasks: dict[int, _EngineTask] = field(default_factory=dict, init=False)
 
     def __post_init__(self) -> None:
-        self._exo_gen = MlxBatchGenerator(
+        self._mlx_gen = MlxBatchGenerator(
             model=self.model,
             stop_tokens=set(eos_ids_from_tokenizer(self.tokenizer)),
             prefill_step_size=4096,
         )
+        self._mlx_gen._needs_topk = False  # pyright: ignore[reportAttributeAccessIssue]
 
     @property
     def has_work(self) -> bool:
         return (
             bool(self._active_tasks)
-            or bool(self._exo_gen.unprocessed_prompts)
-            or self._exo_gen.active_batch is not None
+            or bool(self._mlx_gen.unprocessed_prompts)
+            or self._mlx_gen.active_batch is not None
         )
 
     def submit(
@@ -188,7 +193,7 @@ class ExoBatchGenerator:
 
         max_tokens = task_params.max_output_tokens or MAX_TOKENS
 
-        uids = self._exo_gen.insert(
+        uids = self._mlx_gen.insert(
             prompts=[last_tokens.tolist()],
             max_tokens=[max_tokens],
             caches=[list(cache)],
@@ -211,6 +216,7 @@ class ExoBatchGenerator:
             on_generation_token=on_generation_token,
             generation_start_time=time.perf_counter(),
             prefill_tps=_prefill_tps,
+            generation_time_at_start=self._mlx_gen._stats.generation_time,
         )
 
         return uid
@@ -219,7 +225,12 @@ class ExoBatchGenerator:
         if not self.has_work:
             return []
 
-        responses = self._exo_gen.next()
+        self._mlx_gen._needs_topk = any(  # pyright: ignore[reportAttributeAccessIssue]
+            t.task_params.logprobs for t in self._active_tasks.values()
+        )
+        _step_tic = time.perf_counter()
+        responses = self._mlx_gen.next()
+        _next_elapsed = time.perf_counter() - _step_tic
 
         results: list[tuple[int, GenerationResponse]] = []
 
@@ -277,28 +288,31 @@ class ExoBatchGenerator:
             logprob: float | None = None
             top_logprobs: list[TopLogprobItem] | None = None
             if task_params.logprobs:
-                logprob, top_logprobs = extract_top_logprobs(
-                    logprobs=response.logprobs,
-                    tokenizer=self.tokenizer,
-                    top_logprobs=task_params.top_logprobs or DEFAULT_TOP_LOGPROBS,
-                    selected_token=response.token,
-                )
+                with mx.stream(generation_stream):
+                    logprob, top_logprobs = extract_top_logprobs(
+                        logprobs=response.logprobs,
+                        tokenizer=self.tokenizer,
+                        top_logprobs=task_params.top_logprobs or DEFAULT_TOP_LOGPROBS,
+                        selected_token=response.token,
+                        precomputed_indices=getattr(response, "_topk_indices", None),
+                        precomputed_values=getattr(response, "_topk_values", None),
+                        precomputed_selected=getattr(
+                            response, "_selected_logprob", None
+                        ),
+                    )
 
             stats: GenerationStats | None = None
             usage: Usage | None = None
             if is_done:
-                try:
-                    mlx_stats = self._exo_gen.stats()
-                    generation_tps = mlx_stats.generation_tps
-                except ZeroDivisionError:
-                    generation_elapsed = (
-                        time.perf_counter() - state.generation_start_time
-                    )
-                    generation_tps = (
-                        state.completion_tokens / generation_elapsed
-                        if generation_elapsed > 0
-                        else 0.0
-                    )
+                gen_time_delta = (
+                    self._mlx_gen._stats.generation_time
+                    - state.generation_time_at_start
+                )
+                generation_tps = (
+                    state.completion_tokens / gen_time_delta
+                    if gen_time_delta > 0
+                    else 0.0
+                )
 
                 stats = GenerationStats(
                     prompt_tps=state.prefill_tps,
@@ -345,15 +359,22 @@ class ExoBatchGenerator:
                     -max_stop_len:
                 ]
 
+        _step_elapsed = time.perf_counter() - _step_tic
+        _overhead = _step_elapsed - _next_elapsed
+        if self._mlx_gen._next_count % 64 == 0 and responses:
+            logger.debug(
+                f"step overhead: {_overhead * 1000:.2f}ms (next={_next_elapsed * 1000:.2f}ms total={_step_elapsed * 1000:.2f}ms)"
+            )
+
         return results
 
     def cancel(self, uids: list[int]) -> None:
-        self._exo_gen.remove(uids)
+        self._mlx_gen.remove(uids)
         for uid in uids:
             self._active_tasks.pop(uid, None)
 
     def close(self) -> None:
-        self._exo_gen.close()
+        self._mlx_gen.close()
 
     def _save_prefix_cache(
         self,
@@ -372,9 +393,8 @@ class ExoBatchGenerator:
                 if len(all_prompt_tokens) > 0
                 else 0.0
             )
-            if (
-                matched_index is not None
-                and hit_ratio >= _MIN_PREFIX_HIT_RATIO_TO_UPDATE
+            if matched_index is not None and (
+                prefix_hit_length > 1000 or hit_ratio >= _MIN_PREFIX_HIT_RATIO_TO_UPDATE
             ):
                 self.kv_prefix_cache.update_kv_cache(
                     matched_index,

src/exo/worker/engines/mlx/generator/generate.py

@@ -13,7 +13,7 @@ from mlx_lm.models.cache import ArraysCache, RotatingKVCache
 from mlx_lm.sample_utils import make_logits_processors, make_sampler
 from mlx_lm.tokenizer_utils import TokenizerWrapper
 
-from exo.shared.types.api import (
+from exo.api.types import (
     CompletionTokensDetails,
     FinishReason,
     GenerationStats,
@@ -179,7 +179,8 @@ def pipeline_parallel_prefill(
         flush_prefill_sends()
 
     assert _prompt_cache is not None
-    mx.eval([c.state for c in _prompt_cache])  # type: ignore
+    with mx.stream(generation_stream):
+        mx.eval([c.state for c in _prompt_cache])  # type: ignore
 
     # Final callback matching generate_step
     prompt_progress_callback(total, total)
@@ -312,52 +313,46 @@ def warmup_inference(
     model_id: ModelId,
 ) -> int:
     logger.info(f"warming up inference for instance: {model_id}")
-    t = time.monotonic()
 
     content = "Prompt to warm up the inference engine. Repeat this."
 
+    warmup_task_params = TextGenerationTaskParams(
+        model=model_id,
+        input=[InputMessage(role="user", content=content)],
+        max_output_tokens=50,
+        temperature=0.0,
+    )
+
     warmup_prompt = apply_chat_template(
         tokenizer=tokenizer,
-        task_params=TextGenerationTaskParams(
-            model=ModelId(""),
-            input=[InputMessage(role="user", content=content)],
-        ),
+        task_params=warmup_task_params,
     )
 
     tokens_generated = 0
 
-    cache = make_kv_cache(
-        model=model,
-    )
-
-    # Use a default sampler for warmup
-    sampler = make_sampler(temp=0.0)
-
     mx_barrier(group)
 
     logger.info("Generating warmup tokens")
-    for _r in stream_generate(
+
+    t = time.monotonic()
+
+    for _r in mlx_generate(
         model=model,
         tokenizer=tokenizer,
+        task=warmup_task_params,
         prompt=warmup_prompt,
-        max_tokens=50,
-        sampler=sampler,
-        prompt_cache=cache,
-        prefill_step_size=2048,
-        kv_group_size=KV_GROUP_SIZE,
-        kv_bits=KV_BITS,
+        kv_prefix_cache=None,
+        group=group,
     ):
-        logger.info("Generated warmup token: " + str(_r.text))
         tokens_generated += 1
 
-    logger.info("Generated ALL warmup tokens")
+    check_for_cancel_every = min(
+        math.ceil(tokens_generated / min(time.monotonic() - t, 0.001)), 100
+    )
 
     mx_barrier(group)
 
     logger.info(f"warmed up by generating {tokens_generated} tokens")
-    check_for_cancel_every = min(
-        math.ceil(tokens_generated / min(time.monotonic() - t, 0.001)), 100
-    )
     if group is not None:
         check_for_cancel_every = int(
             mx.max(
@@ -398,52 +393,44 @@ def extract_top_logprobs(
     tokenizer: TokenizerWrapper,
     top_logprobs: int,
     selected_token: int,
+    precomputed_indices: list[int] | None = None,
+    precomputed_values: list[float] | None = None,
+    precomputed_selected: float | None = None,
 ) -> tuple[float, list[TopLogprobItem]]:
-    """Extract the selected token's logprob and top alternative tokens.
-
-    Args:
-        logprobs: Full vocabulary logprobs array from MLX
-        tokenizer: Tokenizer for decoding token IDs to strings
-        top_logprobs: Number of top alternatives to return
-        selected_token: The token ID that was actually sampled
-
-    Returns:
-        Tuple of (selected_token_logprob, list of TopLogprobItem for top alternatives)
-    """
-    # Get the logprob of the selected token
-    selected_logprob = float(logprobs[selected_token].item())
-
-    # Get top indices (most probable tokens)
-    # mx.argpartition gives indices that would partition the array
-    # We negate logprobs since argpartition finds smallest, and we want largest
-    top_logprobs = min(top_logprobs, logprobs.shape[0])  # Don't exceed vocab size
-    top_indices = mx.argpartition(-logprobs, top_logprobs)[:top_logprobs]
-
-    # Get the actual logprob values for these indices
-    top_values = logprobs[top_indices]
-
-    # Sort by logprob (descending) for consistent ordering
-    sort_order = mx.argsort(-top_values)
-    top_indices = top_indices[sort_order]
-    top_values = top_values[sort_order]
+    if (
+        precomputed_indices is not None
+        and precomputed_values is not None
+        and precomputed_selected is not None
+    ):
+        top_indices_list: list[int] = precomputed_indices[:top_logprobs]
+        top_values_list: list[float] = precomputed_values[:top_logprobs]
+        selected_logprob = precomputed_selected
+    else:
+        selected_logprob_arr = logprobs[selected_token]
+        top_logprobs = min(top_logprobs, logprobs.shape[0] - 1)
+        top_indices = mx.argpartition(-logprobs, top_logprobs)[:top_logprobs]
+        top_values = logprobs[top_indices]
+        sort_order = mx.argsort(-top_values)
+        top_indices = top_indices[sort_order]
+        top_values = top_values[sort_order]
+        mx.eval(selected_logprob_arr, top_indices, top_values)
+        selected_logprob = float(selected_logprob_arr.item())
+        top_indices_list = top_indices.tolist()  # type: ignore
+        top_values_list = top_values.tolist()  # type: ignore
 
     # Convert to list of TopLogprobItem
     top_logprob_items: list[TopLogprobItem] = []
-    for i in range(top_logprobs):
-        token_id = int(top_indices[i].item())
-        token_logprob = float(top_values[i].item())
+    for token_id, token_logprob in zip(top_indices_list, top_values_list, strict=True):
         if math.isnan(token_logprob):
             continue
 
         # Decode token ID to string
         token_str = tokenizer.decode([token_id])
-        # Get byte representation
-        token_bytes = list(token_str.encode("utf-8"))
         top_logprob_items.append(
             TopLogprobItem(
                 token=token_str,
                 logprob=token_logprob,
-                bytes=token_bytes,
+                bytes=list(token_str.encode("utf-8")),
             )
         )
 
@@ -624,12 +611,13 @@ def mlx_generate(
         logprob: float | None = None
         top_logprobs: list[TopLogprobItem] | None = None
         if task.logprobs:
-            logprob, top_logprobs = extract_top_logprobs(
-                logprobs=out.logprobs,
-                tokenizer=tokenizer,
-                top_logprobs=task.top_logprobs or DEFAULT_TOP_LOGPROBS,
-                selected_token=out.token,
-            )
+            with mx.stream(generation_stream):
+                logprob, top_logprobs = extract_top_logprobs(
+                    logprobs=out.logprobs,
+                    tokenizer=tokenizer,
+                    top_logprobs=task.top_logprobs or DEFAULT_TOP_LOGPROBS,
+                    selected_token=out.token,
+                )
 
         if is_done:
             # Log generation stats
@@ -657,9 +645,9 @@ def mlx_generate(
                     if len(all_prompt_tokens) > 0
                     else 0.0
                 )
-                if (
-                    matched_index is not None
-                    and hit_ratio >= _MIN_PREFIX_HIT_RATIO_TO_UPDATE
+                if matched_index is not None and (
+                    prefix_hit_length > 1000
+                    or hit_ratio >= _MIN_PREFIX_HIT_RATIO_TO_UPDATE
                 ):
                     kv_prefix_cache.update_kv_cache(
                         matched_index,

src/exo/worker/engines/mlx/patches/__init__.py

@@ -0,0 +1,14 @@
+from exo.worker.engines.mlx.patches.opt_batch_gen import apply_batch_gen_patch
+from exo.worker.engines.mlx.patches.standard_yarn_rope import patch_yarn_rope
+
+_applied = False
+
+
+def apply_mlx_patches() -> None:
+    global _applied
+    if _applied:
+        return
+    _applied = True
+    patch_yarn_rope()
+    # patch_gdn_softplus()
+    apply_batch_gen_patch()

src/exo/worker/engines/mlx/patches/opt_batch_gen.py

@@ -0,0 +1,173 @@
+import time
+from typing import Any, cast
+
+import mlx.core as mx
+from mlx_lm.generate import BatchGenerator, generation_stream
+
+_PRECOMPUTE_TOP_K = 20
+
+_original_public_next = BatchGenerator.next
+
+_pending_topk_idx: mx.array | None = None
+_pending_topk_val: mx.array | None = None
+_pending_selected_lps: mx.array | None = None
+
+
+def _fast_next(self: BatchGenerator) -> list[BatchGenerator.Response]:
+    tic = time.perf_counter()
+    batch = self.active_batch
+    assert batch is not None
+    batch_size = len(batch)
+
+    prev_tokens = batch.y
+    prev_logprobs = batch.logprobs
+
+    has_processors = any(p for ps in batch.logits_processors for p in ps)
+    if has_processors:
+        for i, toks in enumerate(batch.tokens):
+            batch.tokens[i] = mx.concatenate([toks, prev_tokens[i : i + 1]])
+
+    logits = self.model(prev_tokens[:, None], cache=batch.cache)
+    logits = logits[:, -1, :]
+
+    if has_processors:
+        processed_logits: list[mx.array] = []
+        for e in range(batch_size):
+            sample_logits: mx.array = logits[e : e + 1]
+            for processor in batch.logits_processors[e]:
+                sample_logits = processor(batch.tokens[e], sample_logits)
+            processed_logits.append(sample_logits)
+        logits = mx.concatenate(processed_logits, axis=0)
+
+    logprobs = logits - mx.logsumexp(logits, axis=-1, keepdims=True)
+
+    if (
+        batch_size == 1
+        or any(batch.samplers)
+        and all(s is batch.samplers[0] for s in batch.samplers)
+    ):
+        sampler = batch.samplers[0] or self.sampler
+        batch.y = sampler(logprobs)
+    elif any(batch.samplers):
+        all_samples: list[mx.array] = []
+        for e in range(batch_size):
+            s = batch.samplers[e] or self.sampler
+            all_samples.append(s(logprobs[e : e + 1]))
+        batch.y = mx.concatenate(all_samples, axis=0)
+    else:
+        batch.y = self.sampler(logprobs)
+    batch.logprobs = list(logprobs)
+
+    global _pending_topk_idx, _pending_topk_val, _pending_selected_lps
+
+    emit_topk_indices: list[list[int]] = (
+        cast(list[list[int]], _pending_topk_idx.tolist())
+        if _pending_topk_idx is not None
+        else []
+    )
+    emit_topk_values: list[list[float]] = (
+        cast(list[list[float]], _pending_topk_val.tolist())
+        if _pending_topk_val is not None
+        else []
+    )
+    emit_selected_lps: list[float] = (
+        cast(list[float], _pending_selected_lps.tolist())
+        if _pending_selected_lps is not None
+        else []
+    )
+
+    needs_topk: bool = getattr(self, "_needs_topk", False)
+    if needs_topk:
+        k = min(_PRECOMPUTE_TOP_K, logprobs.shape[1])
+        _pending_topk_idx = mx.argpartition(-logprobs, k, axis=1)[:, :k]
+        _pending_topk_val = mx.take_along_axis(logprobs, _pending_topk_idx, axis=1)
+        sort_order = mx.argsort(-_pending_topk_val, axis=1)
+        _pending_topk_idx = mx.take_along_axis(_pending_topk_idx, sort_order, axis=1)
+        _pending_topk_val = mx.take_along_axis(_pending_topk_val, sort_order, axis=1)
+        _pending_selected_lps = logprobs[mx.arange(batch_size), batch.y]
+        mx.async_eval(
+            batch.y,
+            *batch.logprobs,
+            *batch.tokens,
+            _pending_topk_idx,
+            _pending_topk_val,
+            _pending_selected_lps,
+        )
+    else:
+        _pending_topk_idx = None
+        _pending_topk_val = None
+        _pending_selected_lps = None
+        mx.async_eval(batch.y, *batch.logprobs, *batch.tokens)
+
+    prev_token_list: list[int] = cast(list[int], prev_tokens.tolist())
+
+    toc = time.perf_counter()
+    self._stats.generation_time += toc - tic
+
+    keep_idx: list[int] = []
+    end_idx: list[int] = []
+    responses: list[Any] = []
+    stop_tokens = self.stop_tokens
+
+    for e in range(batch_size):
+        t = prev_token_list[e]
+        uid = batch.uids[e]
+        num_tok = batch.num_tokens[e] + 1
+        batch.num_tokens[e] = num_tok
+
+        if t in stop_tokens:
+            finish_reason = "stop"
+            end_idx.append(e)
+        elif num_tok >= batch.max_tokens[e]:
+            finish_reason = "length"
+            end_idx.append(e)
+        else:
+            finish_reason = None
+            keep_idx.append(e)
+
+        cache = None
+        if finish_reason is not None:
+            cache = batch.extract_cache(e)
+        response = self.Response(uid, t, prev_logprobs[e], finish_reason, cache)
+        if emit_topk_indices and e < len(emit_topk_indices):
+            response._topk_indices = emit_topk_indices[e]  # pyright: ignore[reportAttributeAccessIssue]
+            response._topk_values = emit_topk_values[e]  # pyright: ignore[reportAttributeAccessIssue]
+            response._selected_logprob = emit_selected_lps[e]  # pyright: ignore[reportAttributeAccessIssue]
+        responses.append(response)
+
+    if end_idx:
+        if keep_idx:
+            batch.filter(keep_idx)
+            if (
+                _pending_topk_idx is not None
+                and _pending_topk_val is not None
+                and _pending_selected_lps is not None
+            ):
+                ki = mx.array(keep_idx)
+                _pending_topk_idx = _pending_topk_idx[ki]
+                _pending_topk_val = _pending_topk_val[ki]
+                _pending_selected_lps = _pending_selected_lps[ki]
+        else:
+            self.active_batch = None
+            _pending_topk_idx = None
+            _pending_topk_val = None
+            _pending_selected_lps = None
+
+    self._next_count += 1
+    if self._next_count % 512 == 0:
+        mx.clear_cache()
+    self._stats.generation_tokens += len(responses)
+    return responses
+
+
+def _patched_public_next(self: BatchGenerator) -> list[BatchGenerator.Response]:
+    batch = self.active_batch
+    # Only do decode with fast_next
+    if batch is not None and not self.unprocessed_prompts:
+        with mx.stream(generation_stream):
+            return _fast_next(self)
+    return _original_public_next(self)
+
+
+def apply_batch_gen_patch() -> None:
+    BatchGenerator.next = _patched_public_next

src/exo/worker/engines/mlx/patches/standard_yarn_rope.py

@@ -0,0 +1,118 @@
+import math
+
+import mlx.core as mx
+from mlx_lm.models import rope_utils
+
+_original_YarnRoPE_init = rope_utils.YarnRoPE.__init__  # noqa: N816
+_original_initialize_rope = rope_utils.initialize_rope
+
+
+def _patched_yarn_init(
+    self: rope_utils.YarnRoPE,
+    dims: int,
+    traditional: bool = False,
+    max_position_embeddings: int = 2048,
+    base: float = 10000,
+    scaling_factor: float = 1.0,
+    original_max_position_embeddings: int = 4096,
+    beta_fast: float = 32,
+    beta_slow: float = 1,
+    mscale: float = 1,
+    mscale_all_dim: float = 0,
+    truncate: bool = True,
+) -> None:
+    """Patch mlx_lm's YarnRoPE to match vLLM's inverse-frequency blending formula for compatability."""
+
+    super(rope_utils.YarnRoPE, self).__init__()
+
+    def yarn_find_correction_dim(num_rotations: float) -> float:
+        return (
+            dims
+            * math.log(original_max_position_embeddings / (num_rotations * 2 * math.pi))
+        ) / (2 * math.log(base))
+
+    def yarn_find_correction_range() -> tuple[float, float]:
+        low: float = yarn_find_correction_dim(beta_fast)
+        high: float = yarn_find_correction_dim(beta_slow)
+        if truncate:
+            low = math.floor(low)
+            high = math.ceil(high)
+        return max(low, 0), min(high, dims - 1)
+
+    def yarn_get_mscale(scale: float = 1, ms: float = 1) -> float:
+        if scale <= 1:
+            return 1.0
+        return 0.1 * ms * math.log(scale) + 1.0
+
+    def yarn_linear_ramp_mask(min_val: float, max_val: float, dim: int) -> mx.array:
+        if min_val == max_val:
+            max_val += 0.001
+        linear_func = (mx.arange(dim, dtype=mx.float32) - min_val) / (max_val - min_val)
+        return mx.clip(linear_func, 0, 1)
+
+    self.mscale = yarn_get_mscale(scaling_factor, mscale) / yarn_get_mscale(
+        scaling_factor, mscale_all_dim
+    )
+    pos_freqs = base ** (mx.arange(0, dims, 2, dtype=mx.float32) / dims)
+    inv_freq_extrapolation = 1.0 / pos_freqs
+    inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs)
+    low, high = yarn_find_correction_range()
+    inv_freq_mask = 1.0 - yarn_linear_ramp_mask(low, high, dims // 2)
+    inv_freq = (
+        inv_freq_interpolation * (1 - inv_freq_mask)
+        + inv_freq_extrapolation * inv_freq_mask
+    )
+    self._freqs = 1.0 / inv_freq
+    self.dims = dims
+    self.traditional = traditional
+
+
+def _patched_initialize_rope(
+    dims: int,
+    base: float,
+    traditional: bool,
+    scaling_config: dict[str, str | int | float | bool] | None = None,
+    max_position_embeddings: int | None = None,
+) -> object:
+    rope_type = "default"
+    if scaling_config is not None:
+        rope_type = str(
+            scaling_config.get("type") or scaling_config.get("rope_type", "default")
+        )
+
+    # All the yarn rope types supported in mlx lm
+    if rope_type in ("yarn", "deepseek_yarn"):
+        assert scaling_config is not None
+        cfg = scaling_config
+
+        def _float(key: str, default: float) -> float:
+            v = cfg.get(key)
+            return float(v) if v is not None else default
+
+        def _int(key: str, default: int) -> int:
+            v = cfg.get(key)
+            return int(v) if v is not None else default
+
+        return rope_utils.YarnRoPE(
+            dims=dims,
+            max_position_embeddings=max_position_embeddings or 2048,
+            traditional=traditional,
+            scaling_factor=_float("factor", 1.0),
+            base=base,
+            original_max_position_embeddings=_int(
+                "original_max_position_embeddings", 4096
+            ),
+            beta_fast=_float("beta_fast", 32),
+            beta_slow=_float("beta_slow", 1),
+            mscale=_float("mscale", 1),
+            mscale_all_dim=_float("mscale_all_dim", 0),
+        )
+
+    return _original_initialize_rope(
+        dims, base, traditional, scaling_config, max_position_embeddings
+    )
+
+
+def patch_yarn_rope() -> None:
+    rope_utils.YarnRoPE.__init__ = _patched_yarn_init
+    rope_utils.initialize_rope = _patched_initialize_rope

src/exo/worker/engines/mlx/tests/test_extract_top_logprobs.py

@@ -0,0 +1,290 @@
+# type: ignore
+import math
+from unittest.mock import MagicMock
+
+import mlx.core as mx
+import mlx.nn as nn
+import pytest
+from mlx_lm.generate import BatchGenerator
+
+from exo.worker.engines.mlx.generator.generate import extract_top_logprobs
+from exo.worker.engines.mlx.patches.opt_batch_gen import (
+    _PRECOMPUTE_TOP_K,
+    apply_batch_gen_patch,
+)
+
+
+def _mock_tokenizer() -> MagicMock:
+    tok = MagicMock()
+    tok.decode = lambda ids: f"tok_{ids[0]}"
+    return tok
+
+
+def _make_logprobs(values: list[float]) -> mx.array:
+    arr = mx.array(values, dtype=mx.float32)
+    mx.eval(arr)
+    return arr
+
+
+class TestExtractTopLogprobsFallback:
+    def test_returns_correct_selected_logprob(self) -> None:
+        lp = _make_logprobs([-1.0, -2.0, -0.5, -3.0, -4.0])
+        selected, _ = extract_top_logprobs(
+            lp, _mock_tokenizer(), top_logprobs=3, selected_token=2
+        )
+        assert selected == pytest.approx(-0.5)
+
+    def test_returns_top_k_sorted_descending(self) -> None:
+        lp = _make_logprobs([-1.0, -2.0, -0.5, -3.0, -4.0])
+        _, items = extract_top_logprobs(
+            lp, _mock_tokenizer(), top_logprobs=3, selected_token=0
+        )
+        logprob_values = [item.logprob for item in items]
+        assert logprob_values == sorted(logprob_values, reverse=True)
+        assert len(items) == 3
+
+    def test_top_tokens_are_most_probable(self) -> None:
+        lp = _make_logprobs([-5.0, -1.0, -3.0, -0.1, -2.0])
+        _, items = extract_top_logprobs(
+            lp, _mock_tokenizer(), top_logprobs=2, selected_token=0
+        )
+        token_ids = [int(item.token.split("_")[1]) for item in items]
+        assert 3 in token_ids
+        assert 1 in token_ids
+
+    def test_top_logprobs_clamped_to_vocab_size(self) -> None:
+        lp = _make_logprobs([-1.0, -2.0, -3.0, -4.0, -5.0])
+        _, items = extract_top_logprobs(
+            lp, _mock_tokenizer(), top_logprobs=10, selected_token=0
+        )
+        assert len(items) == 4
+
+    def test_nan_logprobs_filtered(self) -> None:
+        lp = _make_logprobs([-1.0, float("nan"), -0.5])
+        _, items = extract_top_logprobs(
+            lp, _mock_tokenizer(), top_logprobs=3, selected_token=0
+        )
+        for item in items:
+            assert not math.isnan(item.logprob)
+
+    def test_token_bytes_correct(self) -> None:
+        tok = MagicMock()
+        tok.decode = lambda ids: "hello"
+        lp = _make_logprobs([-1.0, -2.0])
+        _, items = extract_top_logprobs(lp, tok, top_logprobs=2, selected_token=0)
+        assert items[0].bytes == list("hello".encode("utf-8"))
+
+
+class TestExtractTopLogprobsPrecomputed:
+    def test_uses_precomputed_data(self) -> None:
+        lp = _make_logprobs([-99.0])
+        selected, items = extract_top_logprobs(
+            lp,
+            _mock_tokenizer(),
+            top_logprobs=2,
+            selected_token=0,
+            precomputed_indices=[3, 1, 0],
+            precomputed_values=[-0.1, -1.0, -5.0],
+            precomputed_selected=-0.1,
+        )
+        assert selected == pytest.approx(-0.1)
+        assert len(items) == 2
+        assert items[0].token == "tok_3"
+        assert items[0].logprob == pytest.approx(-0.1)
+        assert items[1].token == "tok_1"
+        assert items[1].logprob == pytest.approx(-1.0)
+
+    def test_slices_precomputed_to_requested_k(self) -> None:
+        lp = _make_logprobs([-99.0])
+        _, items = extract_top_logprobs(
+            lp,
+            _mock_tokenizer(),
+            top_logprobs=1,
+            selected_token=0,
+            precomputed_indices=[3, 1, 0, 2, 4],
+            precomputed_values=[-0.1, -1.0, -2.0, -3.0, -4.0],
+            precomputed_selected=-0.1,
+        )
+        assert len(items) == 1
+        assert items[0].token == "tok_3"
+
+    def test_falls_back_when_precomputed_partial(self) -> None:
+        lp = _make_logprobs([-1.0, -2.0, -0.5])
+        selected, items = extract_top_logprobs(
+            lp,
+            _mock_tokenizer(),
+            top_logprobs=2,
+            selected_token=2,
+            precomputed_indices=[0, 2],
+            precomputed_values=None,
+            precomputed_selected=None,
+        )
+        assert selected == pytest.approx(-0.5)
+        assert len(items) == 2
+
+    def test_precomputed_matches_fallback(self) -> None:
+        lp = _make_logprobs([-1.0, -0.3, -2.5, -0.1, -4.0, -0.8, -3.0, -1.5])
+        tok = _mock_tokenizer()
+
+        selected_fb, items_fb = extract_top_logprobs(
+            lp, tok, top_logprobs=5, selected_token=1
+        )
+
+        pre_indices = [item.token.split("_")[1] for item in items_fb]
+        pre_indices_int = [int(x) for x in pre_indices]
+        pre_values = [item.logprob for item in items_fb]
+
+        selected_pc, items_pc = extract_top_logprobs(
+            lp,
+            tok,
+            top_logprobs=5,
+            selected_token=1,
+            precomputed_indices=pre_indices_int,
+            precomputed_values=pre_values,
+            precomputed_selected=selected_fb,
+        )
+
+        assert selected_pc == pytest.approx(selected_fb)
+        assert len(items_pc) == len(items_fb)
+        for a, b in zip(items_pc, items_fb, strict=True):
+            assert a.token == b.token
+            assert a.logprob == pytest.approx(b.logprob)
+
+
+def _tiny_model() -> nn.Module:
+    from mlx_lm.models.llama import Model, ModelArgs
+
+    mx.random.seed(42)
+    args = ModelArgs(
+        model_type="llama",
+        hidden_size=64,
+        num_hidden_layers=2,
+        intermediate_size=128,
+        num_attention_heads=2,
+        num_key_value_heads=1,
+        rms_norm_eps=1e-6,
+        vocab_size=256,
+        rope_theta=10000.0,
+        tie_word_embeddings=True,
+    )
+    model = Model(args)
+    mx.eval(model.parameters())
+    return model
+
+
+@pytest.mark.slow
+class TestBatchedTopKPrecompute:
+    @pytest.fixture(autouse=True)
+    def _reset_globals(self) -> None:
+        import exo.worker.engines.mlx.patches.opt_batch_gen as _mod
+
+        _mod._pending_topk_idx = None
+        _mod._pending_topk_val = None
+        _mod._pending_selected_lps = None
+
+    def _run_generator(
+        self, model: nn.Module, prompts: list[list[int]], steps: int, needs_topk: bool
+    ) -> list[list[BatchGenerator.Response]]:
+        apply_batch_gen_patch()
+        gen = BatchGenerator(model=model, stop_tokens=set(), prefill_step_size=512)
+        gen._needs_topk = needs_topk
+        gen.insert(prompts)
+        all_responses: list[list[BatchGenerator.Response]] = []
+        for _ in range(steps + len(prompts)):
+            responses = gen.next()
+            if responses:
+                all_responses.append(responses)
+            if gen.active_batch is None and not gen.unprocessed_prompts:
+                break
+        gen.close()
+        return all_responses
+
+    def test_precomputed_topk_attached_to_responses(self) -> None:
+        model = _tiny_model()
+        steps = self._run_generator(model, [[1, 2, 3]], 5, needs_topk=True)
+        found_precomputed = False
+        for step_responses in steps:
+            for resp in step_responses:
+                if hasattr(resp, "_topk_indices"):
+                    found_precomputed = True
+                    assert hasattr(resp, "_topk_values"), (
+                        "Response missing _topk_values"
+                    )
+                    assert hasattr(resp, "_selected_logprob"), (
+                        "Response missing _selected_logprob"
+                    )
+                    assert len(resp._topk_indices) == _PRECOMPUTE_TOP_K
+                    assert len(resp._topk_values) == _PRECOMPUTE_TOP_K
+        assert found_precomputed, "No responses had precomputed topk"
+
+    def test_no_topk_when_not_needed(self) -> None:
+        model = _tiny_model()
+        steps = self._run_generator(model, [[1, 2, 3]], 5, needs_topk=False)
+        for step_responses in steps:
+            for resp in step_responses:
+                assert not hasattr(resp, "_topk_indices")
+
+    def test_precomputed_matches_fallback_in_batch(self) -> None:
+        model = _tiny_model()
+        tok = _mock_tokenizer()
+        steps = self._run_generator(model, [[1, 2, 3]], 10, needs_topk=True)
+        for step_responses in steps[1:]:
+            for resp in step_responses:
+                if not hasattr(resp, "_topk_indices"):
+                    continue
+                selected_fb, items_fb = extract_top_logprobs(
+                    resp.logprobs, tok, top_logprobs=5, selected_token=resp.token
+                )
+                selected_pc, items_pc = extract_top_logprobs(
+                    resp.logprobs,
+                    tok,
+                    top_logprobs=5,
+                    selected_token=resp.token,
+                    precomputed_indices=resp._topk_indices,
+                    precomputed_values=resp._topk_values,
+                    precomputed_selected=resp._selected_logprob,
+                )
+                assert selected_pc == pytest.approx(selected_fb, abs=1e-5)
+                for a, b in zip(items_pc, items_fb, strict=True):
+                    assert a.token == b.token
+                    assert a.logprob == pytest.approx(b.logprob, abs=1e-5)
+
+    def test_topk_correct_after_batch_shrink(self) -> None:
+        model = _tiny_model()
+        tok = _mock_tokenizer()
+        apply_batch_gen_patch()
+        gen = BatchGenerator(
+            model=model, stop_tokens={0}, prefill_step_size=512, max_tokens=3
+        )
+        gen._needs_topk = True
+        gen.insert([[1, 2, 3], [4, 5, 6]], max_tokens=[3, 20])
+
+        seen_shrink = False
+        for _ in range(30):
+            responses = gen.next()
+            for resp in responses:
+                if resp.finish_reason is not None:
+                    seen_shrink = True
+                    continue
+                if not hasattr(resp, "_topk_indices"):
+                    continue
+                selected_fb, items_fb = extract_top_logprobs(
+                    resp.logprobs, tok, top_logprobs=5, selected_token=resp.token
+                )
+                selected_pc, _ = extract_top_logprobs(
+                    resp.logprobs,
+                    tok,
+                    top_logprobs=5,
+                    selected_token=resp.token,
+                    precomputed_indices=resp._topk_indices,
+                    precomputed_values=resp._topk_values,
+                    precomputed_selected=resp._selected_logprob,
+                )
+                assert selected_pc == pytest.approx(selected_fb, abs=1e-5), (
+                    f"Mismatch after batch shrink: precomputed={selected_pc}, fallback={selected_fb}"
+                )
+            if gen.active_batch is None and not gen.unprocessed_prompts:
+                break
+
+        gen.close()
+        assert seen_shrink, "Expected at least one request to finish (batch shrink)"

src/exo/worker/engines/mlx/utils_mlx.py

@@ -486,16 +486,7 @@ def _patch_lossy_chat_template(template: str) -> str | None:
 
 
 def _needs_dsml_encoding(task_params: TextGenerationTaskParams) -> bool:
-    if "deepseek-v3.2" not in task_params.model.lower():
-        return False
-    # Use DSML encoding when tools are provided or tool results are in the conversation
-    if task_params.tools:
-        return True
-    if task_params.chat_template_messages:
-        return any(
-            msg.get("role") == "tool" for msg in task_params.chat_template_messages
-        )
-    return False
+    return "deepseek-v3.2" in task_params.model.lower()
 
 
 def apply_chat_template(
@@ -514,8 +505,6 @@ def apply_chat_template(
     if task_params.chat_template_messages is not None:
         # Use pre-formatted messages that preserve tool_calls, thinking, etc.
         formatted_messages = list(task_params.chat_template_messages)
-        for msg in formatted_messages:
-            _normalize_tool_calls(msg)
     else:
         # Add system message (instructions) if present
         if task_params.instructions:
@@ -541,7 +530,10 @@ def apply_chat_template(
 
         prompt = encode_messages(
             messages=formatted_messages,
-            thinking_mode="thinking" if task_params.enable_thinking else "chat",
+            # Only use chat mode if enable thinking is explicitly Fakse.
+            thinking_mode="chat"
+            if task_params.enable_thinking is False
+            else "thinking",
             tools=task_params.tools,
         )
         if partial_assistant_content:
@@ -549,6 +541,9 @@ def apply_chat_template(
         logger.info(prompt)
         return prompt
 
+    for msg in formatted_messages:
+        _normalize_tool_calls(msg)
+
     extra_kwargs: dict[str, Any] = {}
     if task_params.enable_thinking is not None:
         # Qwen3 and GLM use "enable_thinking"; DeepSeek uses "thinking".
@@ -638,6 +633,7 @@ class NullKVCache(KVCache):
     @property
     def state(self) -> tuple[mx.array, mx.array]:
         # matches what mx.save_safetensors / mx.eval expect
+        assert self.keys is not None and self.values is not None
         return self.keys, self.values
 
     @state.setter
@@ -739,12 +735,9 @@ def _parse_kimi_tool_calls(text: str):
         if func_args_match is None:
             raise ValueError("No tool call arguments found.")
         func_args = func_args_match.group(1)
-        try:
-            arg_dct = json.loads(func_args)  # pyright: ignore[reportAny]
-        except Exception:
-            arg_dct = None
+        arg_dct = json.loads(func_args)  # pyright: ignore[reportAny]
 
-        return dict(id=tool_call_id, name=func_name, arguments=arg_dct)
+        return dict(id=tool_call_id, name=func_name, arguments=arg_dct)  # pyright: ignore[reportAny]
 
     tool_matches = _tool_call_split_regex.findall(text)
     if tool_matches:

src/exo/worker/main.py

@@ -5,10 +5,10 @@ import anyio
 from anyio import fail_after
 from loguru import logger
 
+from exo.api.types import ImageEditsTaskParams
 from exo.download.download_utils import resolve_model_in_path
 from exo.shared.apply import apply
-from exo.shared.models.model_cards import ModelId
-from exo.shared.types.api import ImageEditsTaskParams
+from exo.shared.models.model_cards import ModelId, add_to_card_cache, delete_custom_card
 from exo.shared.types.commands import (
     ForwarderCommand,
     ForwarderDownloadCommand,
@@ -16,6 +16,8 @@ from exo.shared.types.commands import (
 )
 from exo.shared.types.common import CommandId, NodeId, SystemId
 from exo.shared.types.events import (
+    CustomModelCardAdded,
+    CustomModelCardDeleted,
     Event,
     IndexedEvent,
     InputChunkReceived,
@@ -130,6 +132,13 @@ class Worker:
                         event.chunk.data
                     )
 
+                if isinstance(event, CustomModelCardAdded):
+                    await event.model_card.save_to_custom_dir()
+                    add_to_card_cache(event.model_card)
+
+                if isinstance(event, CustomModelCardDeleted):
+                    await delete_custom_card(event.model_id)
+
     async def plan_step(self):
         while True:
             await anyio.sleep(0.1)

src/exo/worker/plan.py

@@ -55,7 +55,7 @@ def plan(
     # Python short circuiting OR logic should evaluate these sequentially.
     return (
         _cancel_tasks(runners, tasks)
-        or _kill_runner(runners, all_runners, instances)
+        or _kill_runner(runners, instances)
         or _create_runner(node_id, runners, instances)
         or _model_needs_download(node_id, runners, global_download_status)
         or _init_distributed_backend(runners, all_runners)
@@ -67,25 +67,14 @@ def plan(
 
 def _kill_runner(
     runners: Mapping[RunnerId, RunnerSupervisor],
-    all_runners: Mapping[RunnerId, RunnerStatus],
     instances: Mapping[InstanceId, Instance],
 ) -> Shutdown | None:
     for runner in runners.values():
         runner_id = runner.bound_instance.bound_runner_id
         if (instance_id := runner.bound_instance.instance.instance_id) not in instances:
             return Shutdown(instance_id=instance_id, runner_id=runner_id)
-
-        for (
-            global_runner_id
-        ) in runner.bound_instance.instance.shard_assignments.node_to_runner.values():
-            if runner_id == global_runner_id:
-                continue
-
-            if isinstance(all_runners.get(global_runner_id, None), RunnerFailed):
-                return Shutdown(
-                    instance_id=instance_id,
-                    runner_id=runner_id,
-                )
+        if isinstance(runner.status, RunnerFailed):
+            return Shutdown(instance_id=instance_id, runner_id=runner_id)
 
 
 def _create_runner(

src/exo/worker/runner/bootstrap.py

@@ -8,6 +8,7 @@ from exo.shared.types.tasks import Task, TaskId
 from exo.shared.types.worker.instances import BoundInstance
 from exo.shared.types.worker.runners import RunnerFailed
 from exo.utils.channels import ClosedResourceError, MpReceiver, MpSender
+from exo.worker.engines.mlx.patches import apply_mlx_patches
 
 logger: "loguru.Logger" = loguru.logger
 
@@ -45,6 +46,8 @@ def entrypoint(
         else:
             from exo.worker.runner.llm_inference.runner import Runner
 
+            apply_mlx_patches()
+
             runner = Runner(
                 bound_instance, event_sender, task_receiver, cancel_receiver
             )

src/exo/worker/runner/image_models/runner.py

@@ -4,10 +4,10 @@ from typing import TYPE_CHECKING, Literal
 
 import mlx.core as mx
 
+from exo.api.types import ImageGenerationStats
 from exo.shared.constants import EXO_MAX_CHUNK_SIZE, EXO_TRACING_ENABLED
 from exo.shared.models.model_cards import ModelTask
 from exo.shared.tracing import clear_trace_buffer, get_trace_buffer
-from exo.shared.types.api import ImageGenerationStats
 from exo.shared.types.chunks import ErrorChunk, ImageChunk
 from exo.shared.types.common import CommandId, ModelId
 from exo.shared.types.events import (
@@ -39,7 +39,6 @@ from exo.shared.types.worker.runner_response import (
 from exo.shared.types.worker.runners import (
     RunnerConnected,
     RunnerConnecting,
-    RunnerFailed,
     RunnerIdle,
     RunnerLoaded,
     RunnerLoading,
@@ -256,9 +255,7 @@ class Runner:
 
     def handle_task(self, task: Task):
         match task:
-            case ConnectToGroup() if isinstance(
-                self.current_status, (RunnerIdle, RunnerFailed)
-            ):
+            case ConnectToGroup() if isinstance(self.current_status, RunnerIdle):
                 logger.info("runner connecting")
                 self.update_status(RunnerConnecting())
                 self.acknowledge_task(task)

src/exo/worker/runner/llm_inference/batch_generator.py

@@ -195,21 +195,29 @@ class SequentialGenerator(InferenceGenerator):
         assert self._active is not None
 
         task, mlx_gen, queue, output_generator = self._active
-        response = None
+        output: list[
+            tuple[TaskId, GenerationResponse | ToolCallResponse | Cancelled | Finished]
+        ] = []
         try:
-            queue.push(next(mlx_gen))
-            response = next(output_generator)
+            response = next(mlx_gen)
+            queue.push(response)
+            # drain potentially many responses every time
+            while (parsed := next(output_generator, None)) is not None:
+                output.append((task.task_id, parsed))
+
         except (StopIteration, PrefillCancelled):
-            response = Finished()
+            output.append((task.task_id, Finished()))
             self._active = None
             if self._queue:
                 self._start_next()
+
         except Exception as e:
             self._send_error(task, e)
             self._active = None
             raise
+
         return itertools.chain(
-            [] if response is None else [(task.task_id, response)],
+            output,
             map(lambda task: (task, Cancelled()), self._cancelled_tasks),
         )
 
@@ -427,11 +435,11 @@ class BatchGenerator(InferenceGenerator):
 
             task, queue, output_generator = self._active_tasks[uid]
             queue.push(response)
-            parsed = next(output_generator)
-
-            if parsed is not None:
+            # If a generator fails to parse for some reason and returns early, we should not crash
+            while (parsed := next(output_generator, None)) is not None:
                 output.append((task.task_id, parsed))
 
+            # check if original response was terminal and append a Finished()
             if response.finish_reason is not None:
                 output.append((task.task_id, Finished()))
                 del self._active_tasks[uid]

src/exo/worker/runner/llm_inference/model_output_parsers.py

@@ -13,7 +13,7 @@ from openai_harmony import (  # pyright: ignore[reportMissingTypeStubs]
     load_harmony_encoding,
 )
 
-from exo.shared.types.api import ToolCallItem
+from exo.api.types import ToolCallItem
 from exo.shared.types.common import ModelId
 from exo.shared.types.mlx import Model
 from exo.shared.types.worker.runner_response import GenerationResponse, ToolCallResponse
@@ -159,11 +159,42 @@ def parse_deepseek_v32(
     # Text accumulated during a tool call block
     tool_call_text = ""
 
+    def _try_parse_tool_call(
+        text: str, response: GenerationResponse
+    ) -> ToolCallResponse | GenerationResponse:
+        parsed = parse_dsml_output(text)
+        if parsed is not None:
+            return ToolCallResponse(
+                tool_calls=parsed, usage=response.usage, stats=response.stats
+            )
+        logger.warning(f"DSML tool call parsing failed for: {text}")
+        return response.model_copy(update={"text": text})
+
     for response in responses:
         if response is None:
             yield None
             continue
 
+        if response.finish_reason is not None:
+            yield from pending_buffer
+            pending_buffer.clear()
+            if in_tool_call:
+                tool_call_text += response.text
+                yield (
+                    _try_parse_tool_call(tool_call_text, response)
+                    if TOOL_CALLS_END in tool_call_text
+                    else response.model_copy(update={"text": tool_call_text})
+                )
+            elif TOOL_CALLS_START in response.text and TOOL_CALLS_END in response.text:
+                dsml_start = response.text.index(TOOL_CALLS_START)
+                before = response.text[:dsml_start]
+                if before:
+                    yield response.model_copy(update={"text": before})
+                yield _try_parse_tool_call(response.text[dsml_start:], response)
+            else:
+                yield response
+            break
+
         # ── Handle thinking tags ──
         if not thinking and THINKING_START in response.text:
             thinking = True
@@ -191,28 +222,7 @@ def parse_deepseek_v32(
         if in_tool_call:
             tool_call_text += response.text
             if TOOL_CALLS_END in tool_call_text:
-                # Parse the accumulated DSML block
-                parsed = parse_dsml_output(tool_call_text)
-                if parsed is not None:
-                    logger.info(f"parsed DSML tool calls: {parsed}")
-                    yield ToolCallResponse(
-                        tool_calls=parsed,
-                        usage=response.usage,
-                        stats=response.stats,
-                    )
-                else:
-                    logger.warning(
-                        f"DSML tool call parsing failed for: {tool_call_text}"
-                    )
-                    yield response.model_copy(update={"text": tool_call_text})
-                in_tool_call = False
-                tool_call_text = ""
-                continue
-
-            # EOS reached before end marker — yield buffered text as-is
-            if response.finish_reason is not None:
-                logger.info("DSML tool call parsing interrupted by EOS")
-                yield response.model_copy(update={"text": tool_call_text})
+                yield _try_parse_tool_call(tool_call_text, response)
                 in_tool_call = False
                 tool_call_text = ""
             continue
@@ -228,33 +238,22 @@ def parse_deepseek_v32(
             if pre_text:
                 # Flush pending buffer tokens that contributed text before the marker
                 for buf_resp in pending_buffer:
-                    if pre_text:
-                        chunk = buf_resp.text
-                        if len(chunk) <= len(pre_text):
-                            yield buf_resp
-                            pre_text = pre_text[len(chunk) :]
-                        else:
-                            yield buf_resp.model_copy(update={"text": pre_text})
-                            pre_text = ""
+                    if not pre_text:
+                        break
+                    chunk = buf_resp.text
+                    if len(chunk) <= len(pre_text):
+                        yield buf_resp
+                        pre_text = pre_text[len(chunk) :]
+                    else:
+                        yield buf_resp.model_copy(update={"text": pre_text})
+                        pre_text = ""
             pending_buffer = []
             tool_call_text = accumulated[start_idx:]
             accumulated = ""
 
             # Check if the end marker is already present (entire tool call in one token)
             if TOOL_CALLS_END in tool_call_text:
-                parsed = parse_dsml_output(tool_call_text)
-                if parsed is not None:
-                    logger.info(f"parsed DSML tool calls: {parsed}")
-                    yield ToolCallResponse(
-                        tool_calls=parsed,
-                        usage=response.usage,
-                        stats=response.stats,
-                    )
-                else:
-                    logger.warning(
-                        f"DSML tool call parsing failed for: {tool_call_text}"
-                    )
-                    yield response.model_copy(update={"text": tool_call_text})
+                yield _try_parse_tool_call(tool_call_text, response)
                 tool_call_text = ""
             else:
                 in_tool_call = True
@@ -267,15 +266,13 @@ def parse_deepseek_v32(
             continue
 
         # No partial match — flush all pending tokens and the current one
-        for buf_resp in pending_buffer:
-            yield buf_resp
-        pending_buffer = []
+        yield from pending_buffer
+        pending_buffer.clear()
         accumulated = ""
         yield response
 
     # Flush any remaining pending buffer at generator end
-    for buf_resp in pending_buffer:
-        yield buf_resp
+    yield from pending_buffer
 
 
 def _could_be_dsml_prefix(text: str) -> bool:
@@ -358,8 +355,10 @@ def parse_tool_calls(
 
             if parsed is None:
                 logger.warning(f"tool call parsing failed for text {combined}")
-                yield response.model_copy(update={"text": combined})
-                continue
+                yield response.model_copy(
+                    update={"text": combined, "token": 0, "finish_reason": "error"}
+                )
+                break
 
             yield ToolCallResponse(
                 tool_calls=parsed, usage=response.usage, stats=response.stats
@@ -374,6 +373,7 @@ def parse_tool_calls(
                 update={
                     "text": "".join(tool_call_text_parts),
                     "token": 0,
+                    "finish_reason": "error",
                 }
             )
             yield response

src/exo/worker/runner/llm_inference/runner.py

@@ -146,9 +146,7 @@ class Runner:
         self.send_task_status(task.task_id, TaskStatus.Running)
 
         match task:
-            case ConnectToGroup() if isinstance(
-                self.current_status, (RunnerIdle, RunnerFailed)
-            ):
+            case ConnectToGroup() if isinstance(self.current_status, RunnerIdle):
                 assert isinstance(self.generator, Builder)
                 logger.info("runner connecting")
                 self.update_status(RunnerConnecting())
@@ -319,7 +317,9 @@ class Runner:
         return ExitCode.AllTasksComplete
 
     def send_response(
-        self, response: GenerationResponse | ToolCallResponse, command_id: CommandId
+        self,
+        response: GenerationResponse | ToolCallResponse,
+        command_id: CommandId,
     ):
         match response:
             case GenerationResponse():

src/exo/worker/runner/llm_inference/tool_parsers.py

@@ -3,7 +3,7 @@ import math
 from dataclasses import dataclass
 from typing import Any, Callable
 
-from exo.shared.types.api import ToolCallItem
+from exo.api.types import ToolCallItem
 
 
 @dataclass

src/exo/worker/runner/runner_supervisor.py

@@ -24,6 +24,7 @@ from exo.shared.types.tasks import (
     CANCEL_ALL_TASKS,
     ImageEdits,
     ImageGeneration,
+    Shutdown,
     Task,
     TaskId,
     TaskStatus,
@@ -110,39 +111,45 @@ class RunnerSupervisor:
 
     async def run(self):
         self.runner_process.start()
-        async with self._tg as tg:
-            tg.start_soon(self._watch_runner)
-            tg.start_soon(self._forward_events)
+        try:
+            async with self._tg as tg:
+                tg.start_soon(self._watch_runner)
+                tg.start_soon(self._forward_events)
+        finally:
+            logger.info("Runner supervisor shutting down")
+            if not self._cancel_watch_runner.cancel_called:
+                self._cancel_watch_runner.cancel()
+            with contextlib.suppress(ClosedResourceError):
+                self._ev_recv.close()
+            with contextlib.suppress(ClosedResourceError):
+                self._task_sender.close()
+            with contextlib.suppress(ClosedResourceError):
+                self._event_sender.close()
+            with contextlib.suppress(ClosedResourceError):
+                self._cancel_sender.send(CANCEL_ALL_TASKS)
+            with contextlib.suppress(ClosedResourceError):
+                self._cancel_sender.close()
+
+            await to_thread.run_sync(self.runner_process.join, 5)
+
+            if self.runner_process.is_alive():
+                logger.warning(
+                    "Runner process didn't shutdown succesfully, terminating"
+                )
+                self.runner_process.terminate()
+                self.runner_process.join(timeout=5)
+                # This is overkill but it's not technically bad, just unnecessary.
+                if self.runner_process.is_alive():
+                    logger.critical("Runner process didn't respond to SIGTERM, killing")
+                    self.runner_process.kill()
+                    self.runner_process.join(timeout=5)
+            else:
+                logger.info("Runner process succesfully terminated")
+
+            self.runner_process.close()
 
     def shutdown(self):
-        logger.info("Runner supervisor shutting down")
         self._tg.cancel_tasks()
-        if not self._cancel_watch_runner.cancel_called:
-            self._cancel_watch_runner.cancel()
-        with contextlib.suppress(ClosedResourceError):
-            self._ev_recv.close()
-        with contextlib.suppress(ClosedResourceError):
-            self._task_sender.close()
-        with contextlib.suppress(ClosedResourceError):
-            self._event_sender.close()
-        with contextlib.suppress(ClosedResourceError):
-            self._cancel_sender.send(CANCEL_ALL_TASKS)
-        with contextlib.suppress(ClosedResourceError):
-            self._cancel_sender.close()
-        self.runner_process.join(5)
-        if not self.runner_process.is_alive():
-            logger.info("Runner process succesfully terminated")
-            return
-
-        # This is overkill but it's not technically bad, just unnecessary.
-        logger.warning("Runner process didn't shutdown succesfully, terminating")
-        self.runner_process.terminate()
-        self.runner_process.join(1)
-        if not self.runner_process.is_alive():
-            return
-
-        logger.critical("Runner process didn't respond to SIGTERM, killing")
-        self.runner_process.kill()
 
     async def start_task(self, task: Task):
         if task.task_id in self.pending:
@@ -163,7 +170,8 @@ class RunnerSupervisor:
             await self._task_sender.send_async(task)
         except ClosedResourceError:
             self.in_progress.pop(task.task_id, None)
-            logger.warning(f"Task {task} dropped, runner closed communication.")
+            if not isinstance(task, Shutdown):
+                logger.warning(f"Task {task} dropped, runner closed communication.")
             return
         await event.wait()
 
@@ -218,12 +226,6 @@ class RunnerSupervisor:
             for tid in self.pending:
                 self.pending[tid].set()
 
-    def __del__(self) -> None:
-        if self.runner_process.is_alive():
-            logger.critical("RunnerSupervisor was not stopped cleanly.")
-            with contextlib.suppress(ValueError):
-                self.runner_process.kill()
-
     async def _watch_runner(self) -> None:
         with self._cancel_watch_runner:
             while True:

src/exo/worker/tests/unittests/test_mlx/test_auto_parallel.py

@@ -43,8 +43,8 @@ def run_pipeline_device(
 
         def __call__(self, x: mx.array, *args: object, **kwargs: object) -> mx.array:
             for layer in self.layers:
-                x = layer(x, *args, **kwargs)  # pyright: ignore[reportUnknownVariableType]
-            return x  # pyright: ignore[reportUnknownVariableType]
+                x = layer(x, *args, **kwargs)
+            return x
 
     try:
         group = mx.distributed.init(backend="ring", strict=True)

src/exo/worker/tests/unittests/test_mlx/test_batch_vs_generate.py

@@ -1,389 +0,0 @@
-import copy
-import gc
-import json
-import shutil
-import tempfile
-from pathlib import Path
-from typing import Any, cast
-
-import mlx.core as mx
-import pytest
-from mlx_lm.tokenizer_utils import TokenizerWrapper
-
-from exo.shared.types.common import ModelId
-from exo.shared.types.mlx import KVCacheType, Model
-from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
-from exo.worker.engines.mlx.cache import CacheSnapshot, KVPrefixCache, cache_length
-from exo.worker.engines.mlx.generator.batch_generate import ExoBatchGenerator
-from exo.worker.engines.mlx.generator.generate import mlx_generate
-from exo.worker.engines.mlx.utils_mlx import (
-    apply_chat_template,
-    load_tokenizer_for_model_id,
-)
-
-from .test_prefix_cache_architectures import (
-    ARCHITECTURES,
-    ArchSpec,
-    _arch_available,  # pyright: ignore[reportPrivateUsage]
-    _build_model,  # pyright: ignore[reportPrivateUsage]
-    _copy_tokenizer,  # pyright: ignore[reportPrivateUsage]
-    _find_snapshot,  # pyright: ignore[reportPrivateUsage]
-    _reduce_config,  # pyright: ignore[reportPrivateUsage]
-)
-
-
-def _make_task(
-    content: str = "Hello, what is 2+2?",
-    max_tokens: int = 10,
-    seed: int = 42,
-) -> TextGenerationTaskParams:
-    return TextGenerationTaskParams(
-        model=ModelId("test"),
-        input=[InputMessage(role="user", content=content)],
-        max_output_tokens=max_tokens,
-        temperature=0.7,
-        seed=seed,
-    )
-
-
-# ── Helpers ──────────────────────────────────────────────────────────────── #
-
-
-def _collect_mlx_generate(
-    model: Model,
-    tokenizer: TokenizerWrapper,
-    task: TextGenerationTaskParams,
-    kv_prefix_cache: KVPrefixCache | None,
-) -> list[int]:
-    """Run mlx_generate and collect output token IDs."""
-    prompt = apply_chat_template(tokenizer=tokenizer, task_params=task)
-    tokens: list[int] = []
-    for resp in mlx_generate(
-        model=model,
-        tokenizer=tokenizer,
-        task=task,
-        prompt=prompt,
-        kv_prefix_cache=kv_prefix_cache,
-        group=None,
-    ):
-        tokens.append(resp.token)
-        if resp.finish_reason is not None:
-            break
-    return tokens
-
-
-def _collect_batch_generate(
-    model: Model,
-    tokenizer: TokenizerWrapper,
-    task_params: TextGenerationTaskParams,
-    kv_prefix_cache: KVPrefixCache | None,
-) -> list[int]:
-    """Run ExoBatchGenerator and collect raw output token IDs"""
-    exo_gen = ExoBatchGenerator(
-        model=model,
-        tokenizer=tokenizer,
-        group=None,
-        kv_prefix_cache=kv_prefix_cache,
-    )
-
-    prompt = apply_chat_template(tokenizer=tokenizer, task_params=task_params)
-    exo_gen.submit(task_params=task_params, prompt=prompt)
-
-    tokens: list[int] = []
-    while exo_gen.has_work:
-        results = exo_gen.step()
-        for _uid, response in results:
-            tokens.append(response.token)
-
-    exo_gen.close()
-    return tokens
-
-
-def _assert_state_equal(sa: object, sb: object, label: str) -> None:
-    """Compare two state items, handling both plain arrays and tuples of arrays (CacheList)."""
-    if isinstance(sa, tuple):
-        assert isinstance(sb, tuple), f"{label}: type mismatch"
-        for k, (arr_a, arr_b) in enumerate(
-            zip(
-                cast(tuple[mx.array, ...], sa),
-                cast(tuple[mx.array, ...], sb),
-                strict=True,
-            )
-        ):
-            a_f = mx.array(arr_a).astype(mx.float32)
-            b_f = mx.array(arr_b).astype(mx.float32)
-            if a_f.size == 0:
-                assert b_f.size == 0, f"{label}[{k}]: size mismatch"
-                continue
-            diff = float(mx.max(mx.abs(a_f - b_f)).item())
-            assert diff == 0.0, f"{label}[{k}]: max diff {diff}"
-    else:
-        sa_f = mx.array(cast(mx.array, sa)).astype(mx.float32)
-        sb_f = mx.array(cast(mx.array, sb)).astype(mx.float32)
-        if sa_f.size == 0:
-            assert sb_f.size == 0, f"{label}: size mismatch"
-            return
-        diff = float(mx.max(mx.abs(sa_f - sb_f)).item())
-        assert diff == 0.0, f"{label}: max diff {diff}"
-
-
-def _compare_cache_arrays(
-    cache_a: KVCacheType,
-    cache_b: KVCacheType,
-    label: str = "",
-) -> None:
-    """Assert two KV caches have identical array values."""
-    assert len(cache_a) == len(cache_b), (
-        f"{label}Cache layer count: {len(cache_a)} vs {len(cache_b)}"
-    )
-    for i, (a, b) in enumerate(zip(cache_a, cache_b, strict=True)):
-        assert type(a) is type(b), (
-            f"{label}Layer {i}: type {type(a).__name__} vs {type(b).__name__}"
-        )
-        states_a = a.state
-        states_b = b.state
-        assert len(states_a) == len(states_b), (
-            f"{label}Layer {i}: state count {len(states_a)} vs {len(states_b)}"
-        )
-        for j, (sa, sb) in enumerate(zip(states_a, states_b, strict=True)):
-            if sa is None and sb is None:
-                continue
-            assert sa is not None and sb is not None, (
-                f"{label}Layer {i}, state {j}: one is None"
-            )
-            _assert_state_equal(sa, sb, f"{label}Layer {i}, state {j}")
-
-
-def _safe_state(cache: object) -> list[object]:
-    """Safely access .state on a cache object. Returns [] if uninitialized."""
-    # RotatingKVCache.state crashes when keys is None (uninitialized)
-    if getattr(cache, "keys", _SENTINEL) is None:
-        return []
-    try:
-        return list(cache.state)  # type: ignore[union-attr]
-    except (AttributeError, TypeError):
-        return []
-
-
-_SENTINEL = object()
-
-
-def _compare_snapshots(
-    snaps_a: list[CacheSnapshot] | None,
-    snaps_b: list[CacheSnapshot] | None,
-    label: str = "",
-) -> None:
-    """Assert two snapshot lists are identical."""
-    if snaps_a is None:
-        assert snaps_b is None, f"{label}One side has snapshots, other doesn't"
-        return
-    assert snaps_b is not None, f"{label}One side has snapshots, other doesn't"
-    assert len(snaps_a) == len(snaps_b), (
-        f"{label}Snapshot count: {len(snaps_a)} vs {len(snaps_b)}"
-    )
-    for k, (sa, sb) in enumerate(zip(snaps_a, snaps_b, strict=True)):
-        assert sa.token_count == sb.token_count, (
-            f"{label}Snapshot {k} token_count: {sa.token_count} vs {sb.token_count}"
-        )
-        for layer_i, (s1, s2) in enumerate(zip(sa.states, sb.states, strict=True)):
-            if s1 is None and s2 is None:
-                continue
-            assert s1 is not None and s2 is not None, (
-                f"{label}Snapshot {k}, layer {layer_i}: one state is None"
-            )
-            state_a = _safe_state(s1)
-            state_b = _safe_state(s2)
-            if not state_a and not state_b:
-                continue
-            assert len(state_a) == len(state_b), (
-                f"{label}Snapshot {k}, layer {layer_i}: state length mismatch"
-            )
-            for st_j, (arr_a, arr_b) in enumerate(zip(state_a, state_b, strict=True)):
-                if arr_a is None and arr_b is None:
-                    continue
-                assert arr_a is not None and arr_b is not None
-                _assert_state_equal(
-                    arr_a,
-                    arr_b,
-                    f"{label}Snapshot {k}, layer {layer_i}, state {st_j}",
-                )
-
-
-# ── Test class ────────────────────────────────────────────────────────────── #
-
-
-@pytest.mark.slow
-class TestBatchVsGenerate:
-    """Verify BatchGenerator matches mlx_generate for output tokens and prefix cache."""
-
-    @pytest.fixture(autouse=True)
-    def _cleanup(self):
-        yield
-        mx.clear_cache()
-        gc.collect()
-
-    @pytest.mark.parametrize(
-        "spec",
-        ARCHITECTURES,
-        ids=[a.name for a in ARCHITECTURES],
-    )
-    def test_same_output_and_cache(self, spec: ArchSpec) -> None:
-        if not _arch_available(spec):
-            pytest.skip(f"Model {spec.hub_name} not cached locally")
-
-        snapshot = _find_snapshot(spec.hub_name)
-        assert snapshot is not None
-
-        tmpdir = Path(tempfile.mkdtemp(prefix=f"exo_batchtest_{spec.name}_"))
-        try:
-            # Build reduced config
-            with open(snapshot / "config.json") as f:
-                cfg = cast(dict[str, Any], json.load(f))
-            reduced = _reduce_config(copy.deepcopy(cfg))
-            (tmpdir / "config.json").write_text(json.dumps(reduced))
-
-            # Copy tokenizer
-            tok_src = snapshot
-            if spec.tokenizer_hub is not None:
-                alt = _find_snapshot(spec.tokenizer_hub)
-                if alt is not None:
-                    tok_src = alt
-            _copy_tokenizer(tok_src, tmpdir)
-
-            # Load tokenizer, build model with random weights
-            model_id = ModelId(f"mlx-community/{spec.hub_name}")
-            tokenizer = load_tokenizer_for_model_id(model_id, tmpdir)
-            mx.random.seed(0)
-            model = _build_model(spec.module, reduced)
-
-            task = _make_task()
-
-            # ── Run mlx_generate path ──
-            # Seed is set inside mlx_generate/ExoBatchGenerator.submit from task.seed
-            kv_mlx = KVPrefixCache(None)
-            mlx_tokens = _collect_mlx_generate(model, tokenizer, task, kv_mlx)
-
-            # ── Run batch generator path ──
-            kv_batch = KVPrefixCache(None)
-            batch_tokens = _collect_batch_generate(model, tokenizer, task, kv_batch)
-
-            # ── Compare output tokens ──
-            assert len(mlx_tokens) > 0, "mlx_generate produced no tokens"
-            assert len(batch_tokens) > 0, "BatchGenerator produced no tokens"
-            assert mlx_tokens == batch_tokens, (
-                f"[{spec.name}] Token mismatch:\n"
-                f"  mlx_generate:    {mlx_tokens}\n"
-                f"  BatchGenerator:  {batch_tokens}"
-            )
-
-            # ── Compare prefix cache KV arrays ──
-            assert len(kv_mlx.caches) == 1, "mlx_generate didn't save to prefix cache"
-            assert len(kv_batch.caches) == 1, (
-                "BatchGenerator didn't save to prefix cache"
-            )
-
-            _compare_cache_arrays(
-                kv_mlx.caches[0],
-                kv_batch.caches[0],
-                label=f"[{spec.name}] ",
-            )
-
-            # ── Compare cache lengths ──
-            mlx_len = cache_length(kv_mlx.caches[0])
-            batch_len = cache_length(kv_batch.caches[0])
-            assert mlx_len == batch_len, (
-                f"[{spec.name}] Cache length: mlx={mlx_len} vs batch={batch_len}"
-            )
-
-            # ── Compare snapshots ──
-            _compare_snapshots(
-                kv_mlx._snapshots[0],  # pyright: ignore[reportPrivateUsage]
-                kv_batch._snapshots[0],  # pyright: ignore[reportPrivateUsage]
-                label=f"[{spec.name}] ",
-            )
-
-        finally:
-            shutil.rmtree(tmpdir, ignore_errors=True)
-
-    @pytest.mark.parametrize(
-        "spec",
-        ARCHITECTURES,
-        ids=[a.name for a in ARCHITECTURES],
-    )
-    def test_concurrent_batch_completes(self, spec: ArchSpec) -> None:
-        """Two requests processed concurrently must both complete without
-        crashing and produce non-empty output.
-
-        Note: batch decode logits are NOT bit-exact with sequential because
-        Metal's matmul kernel picks different reduction tiling for B=1 vs B=2
-        when L=1 (decode step). This introduces sub-ULP float16 diffs in
-        gate_proj/down_proj/lm_head which swiglu amplifies by |up_values|.
-        With random weights these accumulate into argmax flips; with trained
-        weights the diffs are absorbed and output matches exactly (verified
-        with real Llama-3.2-1B-Instruct-4bit weights).
-        """
-        if not _arch_available(spec):
-            pytest.skip(f"Model {spec.hub_name} not cached locally")
-
-        snapshot = _find_snapshot(spec.hub_name)
-        assert snapshot is not None
-
-        tmpdir = Path(tempfile.mkdtemp(prefix=f"exo_concurrent_{spec.name}_"))
-        try:
-            with open(snapshot / "config.json") as f:
-                cfg = cast(dict[str, Any], json.load(f))
-            reduced = _reduce_config(copy.deepcopy(cfg))
-            (tmpdir / "config.json").write_text(json.dumps(reduced))
-
-            tok_src = snapshot
-            if spec.tokenizer_hub is not None:
-                alt = _find_snapshot(spec.tokenizer_hub)
-                if alt is not None:
-                    tok_src = alt
-            _copy_tokenizer(tok_src, tmpdir)
-
-            model_id = ModelId(f"mlx-community/{spec.hub_name}")
-            tokenizer = load_tokenizer_for_model_id(model_id, tmpdir)
-            mx.random.seed(0)
-            model = _build_model(spec.module, reduced)
-
-            # Two different prompts → different prompt lengths.
-            task_a = _make_task(content="Hello, what is 2+2?", seed=42)
-            task_a = task_a.model_copy(update={"temperature": 0.0})
-            task_b = _make_task(
-                content="Write a short poem about the ocean and the sky.",
-                seed=99,
-            )
-            task_b = task_b.model_copy(update={"temperature": 0.0})
-
-            # ── Concurrent: submit both to one ExoBatchGenerator ──
-            exo_gen = ExoBatchGenerator(
-                model=model,
-                tokenizer=tokenizer,
-                group=None,
-                kv_prefix_cache=None,
-            )
-
-            prompt_a = apply_chat_template(tokenizer=tokenizer, task_params=task_a)
-            prompt_b = apply_chat_template(tokenizer=tokenizer, task_params=task_b)
-            uid_a = exo_gen.submit(task_params=task_a, prompt=prompt_a)
-            uid_b = exo_gen.submit(task_params=task_b, prompt=prompt_b)
-
-            batch_tokens: dict[int, list[int]] = {uid_a: [], uid_b: []}
-            finished: set[int] = set()
-            while exo_gen.has_work:
-                results = exo_gen.step()
-                for uid, response in results:
-                    batch_tokens[uid].append(response.token)
-                    if response.finish_reason is not None:
-                        finished.add(uid)
-
-            exo_gen.close()
-
-            # ── Verify both completed ──
-            assert len(batch_tokens[uid_a]) > 0, "No tokens for task A"
-            assert len(batch_tokens[uid_b]) > 0, "No tokens for task B"
-            assert uid_a in finished, "Task A never finished"
-            assert uid_b in finished, "Task B never finished"
-        finally:
-            shutil.rmtree(tmpdir, ignore_errors=True)

src/exo/worker/tests/unittests/test_mlx/test_prefix_cache_architectures.py

@@ -190,7 +190,7 @@ ARCHITECTURES: list[ArchSpec] = [
 
 def _arch_available(spec: ArchSpec) -> bool:
     snap = _find_snapshot(spec.hub_name)
-    if snap is None:
+    if snap is None or not (snap / "config.json").exists():
         return False
     if spec.tokenizer_hub is not None:
         return _find_snapshot(spec.tokenizer_hub) is not None

src/exo/worker/tests/unittests/test_runner/test_dsml_e2e.py

@@ -2,6 +2,7 @@ import json
 from collections.abc import Generator
 from typing import Any
 
+from exo.shared.types.common import ModelId
 from exo.shared.types.worker.runner_response import (
     GenerationResponse,
     ToolCallResponse,
@@ -965,3 +966,70 @@ class TestE2EFullRoundTrip:
         assert "sunny" in final_text.lower()
         assert "5°C" in final_text
         assert "12°C" in final_text
+
+
+class TestMultiTurnThinkingPrompt:
+    def test_no_orphan_think_end_in_multiturn(self):
+        messages: list[dict[str, Any]] = [
+            {"role": "user", "content": "Hi!"},
+            {"role": "assistant", "content": "Hello! How can I help you today?"},
+            {"role": "user", "content": "Tell me about Paris."},
+        ]
+        prompt = encode_messages(messages, thinking_mode="thinking")
+        assistant_token = "<\uff5cAssistant\uff5c>"
+        parts = prompt.split(assistant_token)
+        for part in parts[1:]:
+            assert not part.startswith(THINKING_END), (
+                f"Orphan </think> without <think> after <Assistant>: ...{assistant_token}{part[:50]}"
+            )
+
+
+class TestApplyChatTemplateWithToolCalls:
+    def test_dsml_encoding_with_tool_calls_in_history(self):
+        from exo.shared.types.text_generation import (
+            InputMessage,
+            TextGenerationTaskParams,
+        )
+        from exo.worker.engines.mlx.utils_mlx import apply_chat_template
+
+        chat_template_messages: list[dict[str, Any]] = [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "What's the weather?"},
+            {
+                "role": "assistant",
+                "content": "",
+                "tool_calls": [
+                    {
+                        "id": "call_1",
+                        "type": "function",
+                        "function": {
+                            "name": "get_weather",
+                            "arguments": '{"city": "Tokyo"}',
+                        },
+                    }
+                ],
+            },
+            {"role": "tool", "content": "Sunny, 25°C"},
+            {"role": "user", "content": "Thanks!"},
+        ]
+
+        from unittest.mock import MagicMock
+
+        tokenizer = MagicMock()
+        tokenizer.has_thinking = True
+        tokenizer.think_start = "<think>"
+        tokenizer.think_end = "</think>"
+
+        params = TextGenerationTaskParams(
+            model=ModelId("mlx-community/DeepSeek-V3.2-8bit"),
+            input=[InputMessage(role="user", content="Thanks!")],
+            instructions="You are a helpful assistant.",
+            enable_thinking=True,
+            chat_template_messages=chat_template_messages,
+            tools=_WEATHER_TOOLS,
+        )
+
+        prompt = apply_chat_template(tokenizer, params)
+        assert "get_weather" in prompt
+        assert "Tokyo" in prompt
+        assert "Sunny" in prompt

src/exo/worker/tests/unittests/test_runner/test_finish_reason_sse.py

@@ -0,0 +1,332 @@
+from collections.abc import Generator
+from typing import Any
+
+from exo.shared.types.worker.runner_response import (
+    FinishReason,
+    GenerationResponse,
+    ToolCallResponse,
+)
+from exo.worker.engines.mlx.dsml_encoding import (
+    DSML_TOKEN,
+    THINKING_END,
+    THINKING_START,
+    TOOL_CALLS_END,
+    TOOL_CALLS_START,
+)
+from exo.worker.runner.llm_inference.model_output_parsers import (
+    parse_deepseek_v32,
+    parse_thinking_models,
+    parse_tool_calls,
+)
+from exo.worker.runner.llm_inference.tool_parsers import make_mlx_parser
+
+
+def _make_response(
+    text: str, token: int, finish_reason: FinishReason | None = None
+) -> GenerationResponse:
+    return GenerationResponse(
+        text=text, token=token, finish_reason=finish_reason, usage=None
+    )
+
+
+def _queue_source(
+    tokens: list[GenerationResponse],
+) -> Generator[GenerationResponse | None]:
+    for token in tokens:
+        yield token
+        yield None
+    while True:
+        yield None
+
+
+def _step_until_finish(
+    parser_gen: Generator[GenerationResponse | ToolCallResponse | None],
+    max_steps: int = 200,
+) -> list[GenerationResponse | ToolCallResponse]:
+    results: list[GenerationResponse | ToolCallResponse] = []
+    for _ in range(max_steps):
+        try:
+            result = next(parser_gen)
+        except StopIteration:
+            break
+        if result is None:
+            continue
+        results.append(result)
+        if isinstance(result, GenerationResponse) and result.finish_reason is not None:
+            return results
+        if isinstance(result, ToolCallResponse):
+            return results
+    return results
+
+
+def _got_finish(results: list[GenerationResponse | ToolCallResponse]) -> bool:
+    for r in results:
+        if isinstance(r, ToolCallResponse):
+            return True
+        if r.finish_reason is not None:
+            return True
+    return False
+
+
+# ── parse_deepseek_v32 ──────────────────────────────────────────
+
+
+class TestDeepSeekV32FinishReason:
+    def test_finish_reason_with_buffered_dsml_prefix(self):
+        tokens = [
+            _make_response("Hello! The answer is x", 0),
+            _make_response("<", 1),
+            _make_response("", 2, finish_reason="stop"),
+        ]
+        results = _step_until_finish(parse_deepseek_v32(_queue_source(tokens)))
+        assert _got_finish(results)
+        full_text = "".join(
+            r.text for r in results if isinstance(r, GenerationResponse)
+        )
+        assert "Hello" in full_text
+        assert "<" in full_text
+
+    def test_finish_reason_completes_tool_call_block(self):
+        tokens = [
+            _make_response(TOOL_CALLS_START, 0),
+            _make_response("\n", 1),
+            _make_response(f'<{DSML_TOKEN}invoke name="get_weather">\n', 2),
+            _make_response(
+                f'<{DSML_TOKEN}parameter name="city" string="true">Tokyo</{DSML_TOKEN}parameter>\n',
+                3,
+            ),
+            _make_response(f"</{DSML_TOKEN}invoke>\n", 4),
+            _make_response(TOOL_CALLS_END, 5, finish_reason="stop"),
+        ]
+        results = _step_until_finish(parse_deepseek_v32(_queue_source(tokens)))
+        tool_results = [r for r in results if isinstance(r, ToolCallResponse)]
+        assert len(tool_results) == 1
+        assert tool_results[0].tool_calls[0].name == "get_weather"
+
+    def test_finish_reason_mid_tool_call_before_close(self):
+        tokens = [
+            _make_response(TOOL_CALLS_START, 0),
+            _make_response("\n", 1),
+            _make_response(
+                f'<{DSML_TOKEN}invoke name="get_weather">\n', 2, finish_reason="stop"
+            ),
+        ]
+        results = _step_until_finish(parse_deepseek_v32(_queue_source(tokens)))
+        assert _got_finish(results)
+
+    def test_finish_reason_single_token_complete_dsml_block(self):
+        dsml_block = (
+            f"{TOOL_CALLS_START}\n"
+            f'<{DSML_TOKEN}invoke name="get_weather">\n'
+            f'<{DSML_TOKEN}parameter name="city" string="true">Tokyo</{DSML_TOKEN}parameter>\n'
+            f"</{DSML_TOKEN}invoke>\n"
+            f"{TOOL_CALLS_END}"
+        )
+        tokens = [_make_response(dsml_block, 0, finish_reason="stop")]
+        results = _step_until_finish(parse_deepseek_v32(_queue_source(tokens)))
+        tool_results = [r for r in results if isinstance(r, ToolCallResponse)]
+        assert len(tool_results) == 1
+        assert tool_results[0].tool_calls[0].name == "get_weather"
+
+    def test_finish_reason_during_thinking(self):
+        tokens = [
+            _make_response(THINKING_START, 0),
+            _make_response("I need to think about this", 1),
+            _make_response(" carefully before responding", 2, finish_reason="stop"),
+        ]
+        results = _step_until_finish(parse_deepseek_v32(_queue_source(tokens)))
+        assert _got_finish(results)
+
+    def test_finish_reason_after_thinking_then_tool_call(self):
+        tokens = [
+            _make_response(THINKING_START, 0),
+            _make_response("Let me check the weather.", 1),
+            _make_response(THINKING_END, 2),
+            _make_response("\n\n", 3),
+            _make_response(TOOL_CALLS_START, 4),
+            _make_response("\n", 5),
+            _make_response(f'<{DSML_TOKEN}invoke name="get_weather">\n', 6),
+            _make_response(
+                f'<{DSML_TOKEN}parameter name="city" string="true">NYC</{DSML_TOKEN}parameter>\n',
+                7,
+            ),
+            _make_response(f"</{DSML_TOKEN}invoke>\n", 8),
+            _make_response(TOOL_CALLS_END, 9, finish_reason="stop"),
+        ]
+        results = _step_until_finish(parse_deepseek_v32(_queue_source(tokens)))
+        tool_results = [r for r in results if isinstance(r, ToolCallResponse)]
+        assert len(tool_results) == 1
+        assert tool_results[0].tool_calls[0].name == "get_weather"
+
+    def test_finish_reason_normal_text_no_buffering(self):
+        tokens = [
+            _make_response("Hello", 0),
+            _make_response(" world", 1),
+            _make_response("!", 2, finish_reason="stop"),
+        ]
+        results = _step_until_finish(parse_deepseek_v32(_queue_source(tokens)))
+        assert _got_finish(results)
+        full_text = "".join(
+            r.text for r in results if isinstance(r, GenerationResponse)
+        )
+        assert full_text == "Hello world!"
+
+    def test_finish_reason_multiple_buffered_prefix_tokens(self):
+        tokens = [
+            _make_response("text ", 0),
+            _make_response("<", 1),
+            _make_response("not a tag", 2),
+            _make_response(" more<", 3),
+            _make_response("", 4, finish_reason="stop"),
+        ]
+        results = _step_until_finish(parse_deepseek_v32(_queue_source(tokens)))
+        assert _got_finish(results)
+
+
+# ── parse_thinking_models ────────────────────────────────────────
+
+
+class TestThinkingModelsFinishReason:
+    def test_finish_reason_during_thinking(self):
+        tokens = [
+            _make_response("<think>", 0),
+            _make_response("reasoning here", 1),
+            _make_response("more reasoning", 2, finish_reason="stop"),
+        ]
+        results = _step_until_finish(
+            parse_thinking_models(
+                _queue_source(tokens),
+                think_start="<think>",
+                think_end="</think>",
+                starts_in_thinking=False,
+            )
+        )
+        assert _got_finish(results)
+        last_gen = [
+            r
+            for r in results
+            if isinstance(r, GenerationResponse) and r.finish_reason is not None
+        ]
+        assert len(last_gen) == 1
+        assert last_gen[0].is_thinking is False
+
+    def test_finish_reason_after_thinking(self):
+        tokens = [
+            _make_response("<think>", 0),
+            _make_response("hmm", 1),
+            _make_response("</think>", 2),
+            _make_response("The answer is 42.", 3, finish_reason="stop"),
+        ]
+        results = _step_until_finish(
+            parse_thinking_models(
+                _queue_source(tokens),
+                think_start="<think>",
+                think_end="</think>",
+                starts_in_thinking=False,
+            )
+        )
+        assert _got_finish(results)
+
+    def test_finish_reason_starts_in_thinking(self):
+        tokens = [
+            _make_response("still thinking", 0),
+            _make_response("</think>", 1),
+            _make_response("done", 2, finish_reason="stop"),
+        ]
+        results = _step_until_finish(
+            parse_thinking_models(
+                _queue_source(tokens),
+                think_start="<think>",
+                think_end="</think>",
+                starts_in_thinking=True,
+            )
+        )
+        assert _got_finish(results)
+
+
+# ── parse_tool_calls (generic) ──────────────────────────────────
+
+
+def _dummy_parser_fn(text: str) -> dict[str, Any]:
+    return {"name": "test_fn", "arguments": {"arg": text}}
+
+
+_dummy_parser = make_mlx_parser("<tool_call>", "</tool_call>", _dummy_parser_fn)
+
+
+class TestGenericToolCallsFinishReason:
+    def test_finish_reason_after_complete_tool_call(self):
+        tokens = [
+            _make_response("<tool_call>", 0),
+            _make_response("body", 1),
+            _make_response("</tool_call>", 2),
+            _make_response("extra text", 3, finish_reason="stop"),
+        ]
+        results = _step_until_finish(
+            parse_tool_calls(
+                _queue_source(tokens),
+                _dummy_parser,
+                tools=None,
+            )
+        )
+        tool_results = [r for r in results if isinstance(r, ToolCallResponse)]
+        assert len(tool_results) == 1
+
+    def test_finish_reason_mid_tool_call_unclosed(self):
+        tokens = [
+            _make_response("<tool_call>", 0),
+            _make_response("partial content", 1, finish_reason="stop"),
+        ]
+        results = _step_until_finish(
+            parse_tool_calls(
+                _queue_source(tokens),
+                _dummy_parser,
+                tools=None,
+            )
+        )
+        assert _got_finish(results)
+
+    def test_finish_reason_no_tool_calls(self):
+        tokens = [
+            _make_response("Just", 0),
+            _make_response(" a", 1),
+            _make_response(" normal", 2),
+            _make_response(" response.", 3, finish_reason="stop"),
+        ]
+        results = _step_until_finish(
+            parse_tool_calls(
+                _queue_source(tokens),
+                _dummy_parser,
+                tools=None,
+            )
+        )
+        assert _got_finish(results)
+
+
+# ── Double parser chain (parse_thinking_models → parse_deepseek_v32) ──
+
+
+class TestBatchGeneratorSingleNext:
+    def test_finish_reason_with_buffered_tokens_drain_loop(self):
+        from exo.worker.runner.llm_inference.batch_generator import GeneratorQueue
+
+        queue: GeneratorQueue[GenerationResponse] = GeneratorQueue()
+        parser = parse_deepseek_v32(queue.gen())
+
+        tokens = [
+            _make_response("Hello ", 0),
+            _make_response(" `<", 1),
+            _make_response("", 2, finish_reason="stop"),
+        ]
+
+        collected: list[GenerationResponse | ToolCallResponse] = []
+        for token in tokens:
+            queue.push(token)
+            while (parsed := next(parser, None)) is not None:
+                collected.append(parsed)
+            if token.finish_reason is not None:
+                break
+
+        assert _got_finish(collected), (
+            f"No finish_reason in collected: {[(type(r).__name__, getattr(r, 'finish_reason', None) if isinstance(r, GenerationResponse) else 'tool') for r in collected]}"
+        )

src/exo/worker/tests/unittests/test_runner/test_parse_gpt_oss.py

@@ -1,6 +1,6 @@
 from collections.abc import Generator
 
-from exo.shared.types.api import FinishReason
+from exo.api.types import FinishReason
 from exo.shared.types.worker.runner_response import (
     GenerationResponse,
     ToolCallResponse,

src/exo/worker/tests/unittests/test_runner/test_parse_tool_calls.py

@@ -87,6 +87,7 @@ class TestParseToolCalls:
         assert len(results) == 1
         assert isinstance(results[0], GenerationResponse)
         assert results[0].text == "<tool_call>bad content</tool_call>"
+        assert results[0].finish_reason == "error"
 
     def test_tool_schema_coerces_string_arguments_to_expected_types(self):
         """Tool argument values should be coerced using provided JSON schema."""

tests/auto_bench.sh

@@ -50,6 +50,6 @@ bench_runner="${hosts[0]}"
 mkdir -p "./bench/$commit"
 nix run .#exo-get-all-models-on-cluster -- "$bench_runner" | while IFS= read -r model; do
   echo "running bench for $model" 1>&2
-  ssh -Tn -o BatchMode=yes -o ServerAliveInterval=30 "$bench_runner@$bench_runner" "/nix/var/nix/profiles/default/bin/nix run github:exo-explore/exo/$commit#exo-bench -- --model $model --pp 128 4096 --tg 128 --stdout --skip-tensor-ring" >>"./bench/$commit/${model//\//--}.json"
+  ssh -Tn -o BatchMode=yes -o ServerAliveInterval=30 "$bench_runner@$bench_runner" "/nix/var/nix/profiles/default/bin/nix run github:exo-explore/exo/$commit#exo-bench -- --model $model --pp 128 4096 --tg 128 --concurrency 1 3 8 --stdout --skip-tensor-ring" >>"./bench/$commit/${model//\//--}.json"
   echo
 done

tests/run_exo_on.sh

@@ -11,10 +11,17 @@ set -euo pipefail
   exit 1
 }
 
+upstream=$(git rev-parse --abbrev-ref --symbolic-full-name "@{u}" 2>/dev/null) || {
+  echo "No upstream"
+  exit 1
+}
 commit=$(git rev-parse HEAD)
-git fetch -q origin
-git branch -r --contains "$commit" | grep -qE '^\s*origin/' || {
-  echo "Not pushed to origin"
+remote=${upstream%%/*}
+remote_installable=$(git remote get-url "$remote" | sed -E "s#^(git@github.com:|https://github\.com/)([^/]+)/([^/]+)(\.git)?\$#github:\2/\3/$commit#")
+
+git fetch -q "$remote"
+git branch -r --contains "$commit" | grep -qE "^[[:space:]]*$remote/" || {
+  echo "Not pushed to $remote"
   exit 1
 }
 
@@ -35,7 +42,7 @@ i=0
 for host; do
   colour=${colours[i++ % 4]}
   ssh -T -o BatchMode=yes -o ServerAliveInterval=30 "$host@$host" \
-    "EXO_LIBP2P_NAMESPACE=$commit /nix/var/nix/profiles/default/bin/nix run github:exo-explore/exo/$commit" |&
+    "EXO_LIBP2P_NAMESPACE=$commit /nix/var/nix/profiles/default/bin/nix run $remote_installable" 2>&1 |
     awk -v p="${colour}[${host}]${reset}" '{ print p $0; fflush() }' &
 done
 

uv.lock

@@ -213,14 +213,20 @@ sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8
 wheels = [
     { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" },
     { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" },
+    { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" },
+    { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" },
     { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" },
     { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" },
     { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" },
     { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" },
+    { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" },
+    { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" },
     { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" },
     { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" },
     { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" },
     { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" },
+    { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" },
+    { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" },
     { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" },
     { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" },
     { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" },
@@ -344,8 +350,10 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5c/49/498c86566a1d80e978b42f0d702795f69887005548c041636df6ae1ca64c/cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d", size = 4450807, upload-time = "2025-10-15T23:16:56.414Z" },
     { url = "https://files.pythonhosted.org/packages/4b/0a/863a3604112174c8624a2ac3c038662d9e59970c7f926acdcfaed8d61142/cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb", size = 4299615, upload-time = "2025-10-15T23:16:58.442Z" },
     { url = "https://files.pythonhosted.org/packages/64/02/b73a533f6b64a69f3cd3872acb6ebc12aef924d8d103133bb3ea750dc703/cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849", size = 4016800, upload-time = "2025-10-15T23:17:00.378Z" },
+    { url = "https://files.pythonhosted.org/packages/25/d5/16e41afbfa450cde85a3b7ec599bebefaef16b5c6ba4ec49a3532336ed72/cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8", size = 4984707, upload-time = "2025-10-15T23:17:01.98Z" },
     { url = "https://files.pythonhosted.org/packages/c9/56/e7e69b427c3878352c2fb9b450bd0e19ed552753491d39d7d0a2f5226d41/cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec", size = 4482541, upload-time = "2025-10-15T23:17:04.078Z" },
     { url = "https://files.pythonhosted.org/packages/78/f6/50736d40d97e8483172f1bb6e698895b92a223dba513b0ca6f06b2365339/cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91", size = 4299464, upload-time = "2025-10-15T23:17:05.483Z" },
+    { url = "https://files.pythonhosted.org/packages/00/de/d8e26b1a855f19d9994a19c702fa2e93b0456beccbcfe437eda00e0701f2/cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e", size = 4950838, upload-time = "2025-10-15T23:17:07.425Z" },
     { url = "https://files.pythonhosted.org/packages/8f/29/798fc4ec461a1c9e9f735f2fc58741b0daae30688f41b2497dcbc9ed1355/cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926", size = 4481596, upload-time = "2025-10-15T23:17:09.343Z" },
     { url = "https://files.pythonhosted.org/packages/15/8d/03cd48b20a573adfff7652b76271078e3045b9f49387920e7f1f631d125e/cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71", size = 4426782, upload-time = "2025-10-15T23:17:11.22Z" },
     { url = "https://files.pythonhosted.org/packages/fa/b1/ebacbfe53317d55cf33165bda24c86523497a6881f339f9aae5c2e13e57b/cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac", size = 4698381, upload-time = "2025-10-15T23:17:12.829Z" },
@@ -353,8 +361,10 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/c5/fd/bc1daf8230eaa075184cbbf5f8cd00ba9db4fd32d63fb83da4671b72ed8a/cryptography-46.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715", size = 4435078, upload-time = "2025-10-15T23:17:23.042Z" },
     { url = "https://files.pythonhosted.org/packages/82/98/d3bd5407ce4c60017f8ff9e63ffee4200ab3e23fe05b765cab805a7db008/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54", size = 4293460, upload-time = "2025-10-15T23:17:24.885Z" },
     { url = "https://files.pythonhosted.org/packages/26/e9/e23e7900983c2b8af7a08098db406cf989d7f09caea7897e347598d4cd5b/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459", size = 3995237, upload-time = "2025-10-15T23:17:26.449Z" },
+    { url = "https://files.pythonhosted.org/packages/91/15/af68c509d4a138cfe299d0d7ddb14afba15233223ebd933b4bbdbc7155d3/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422", size = 4967344, upload-time = "2025-10-15T23:17:28.06Z" },
     { url = "https://files.pythonhosted.org/packages/ca/e3/8643d077c53868b681af077edf6b3cb58288b5423610f21c62aadcbe99f4/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7", size = 4466564, upload-time = "2025-10-15T23:17:29.665Z" },
     { url = "https://files.pythonhosted.org/packages/0e/43/c1e8726fa59c236ff477ff2b5dc071e54b21e5a1e51aa2cee1676f1c986f/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044", size = 4292415, upload-time = "2025-10-15T23:17:31.686Z" },
+    { url = "https://files.pythonhosted.org/packages/42/f9/2f8fefdb1aee8a8e3256a0568cffc4e6d517b256a2fe97a029b3f1b9fe7e/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665", size = 4931457, upload-time = "2025-10-15T23:17:33.478Z" },
     { url = "https://files.pythonhosted.org/packages/79/30/9b54127a9a778ccd6d27c3da7563e9f2d341826075ceab89ae3b41bf5be2/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3", size = 4466074, upload-time = "2025-10-15T23:17:35.158Z" },
     { url = "https://files.pythonhosted.org/packages/ac/68/b4f4a10928e26c941b1b6a179143af9f4d27d88fe84a6a3c53592d2e76bf/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20", size = 4420569, upload-time = "2025-10-15T23:17:37.188Z" },
     { url = "https://files.pythonhosted.org/packages/a3/49/3746dab4c0d1979888f125226357d3262a6dd40e114ac29e3d2abdf1ec55/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de", size = 4681941, upload-time = "2025-10-15T23:17:39.236Z" },
@@ -362,8 +372,10 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/26/42/fa8389d4478368743e24e61eea78846a0006caffaf72ea24a15159215a14/cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d", size = 4440029, upload-time = "2025-10-15T23:17:49.837Z" },
     { url = "https://files.pythonhosted.org/packages/5f/eb/f483db0ec5ac040824f269e93dd2bd8a21ecd1027e77ad7bdf6914f2fd80/cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0", size = 4297222, upload-time = "2025-10-15T23:17:51.357Z" },
     { url = "https://files.pythonhosted.org/packages/fd/cf/da9502c4e1912cb1da3807ea3618a6829bee8207456fbbeebc361ec38ba3/cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc", size = 4012280, upload-time = "2025-10-15T23:17:52.964Z" },
+    { url = "https://files.pythonhosted.org/packages/6b/8f/9adb86b93330e0df8b3dcf03eae67c33ba89958fc2e03862ef1ac2b42465/cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3", size = 4978958, upload-time = "2025-10-15T23:17:54.965Z" },
     { url = "https://files.pythonhosted.org/packages/d1/a0/5fa77988289c34bdb9f913f5606ecc9ada1adb5ae870bd0d1054a7021cc4/cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971", size = 4473714, upload-time = "2025-10-15T23:17:56.754Z" },
     { url = "https://files.pythonhosted.org/packages/14/e5/fc82d72a58d41c393697aa18c9abe5ae1214ff6f2a5c18ac470f92777895/cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac", size = 4296970, upload-time = "2025-10-15T23:17:58.588Z" },
+    { url = "https://files.pythonhosted.org/packages/78/06/5663ed35438d0b09056973994f1aec467492b33bd31da36e468b01ec1097/cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04", size = 4940236, upload-time = "2025-10-15T23:18:00.897Z" },
     { url = "https://files.pythonhosted.org/packages/fc/59/873633f3f2dcd8a053b8dd1d38f783043b5fce589c0f6988bf55ef57e43e/cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506", size = 4472642, upload-time = "2025-10-15T23:18:02.749Z" },
     { url = "https://files.pythonhosted.org/packages/3d/39/8e71f3930e40f6877737d6f69248cf74d4e34b886a3967d32f919cc50d3b/cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963", size = 4423126, upload-time = "2025-10-15T23:18:04.85Z" },
     { url = "https://files.pythonhosted.org/packages/cd/c7/f65027c2810e14c3e7268353b1681932b87e5a48e65505d8cc17c99e36ae/cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4", size = 4686573, upload-time = "2025-10-15T23:18:06.908Z" },
@@ -473,7 +485,7 @@ dependencies = [
     { name = "loguru", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "mflux", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "mlx", version = "0.30.6", source = { registry = "https://pypi.org/simple" }, extra = ["cpu"], marker = "sys_platform == 'linux'" },
-    { name = "mlx", version = "0.30.7.dev20260303+257d5692", source = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks#257d5692fc7af6bba3b8afaeb63c549b7d1e43d5" }, marker = "sys_platform == 'darwin'" },
+    { name = "mlx", version = "0.31.2.dev20260324+e5e64331", source = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks#e5e64331830d9b04ae9082b843073f9c1fa7705e" }, marker = "sys_platform == 'darwin'" },
     { name = "mlx-lm", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "msgspec", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "openai-harmony", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -509,10 +521,10 @@ requires-dist = [
     { name = "huggingface-hub", specifier = ">=0.33.4" },
     { name = "hypercorn", specifier = ">=0.18.0" },
     { name = "loguru", specifier = ">=0.7.3" },
-    { name = "mflux", specifier = "==0.16.9" },
+    { name = "mflux", specifier = "==0.17.2" },
     { name = "mlx", marker = "sys_platform == 'darwin'", git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks" },
     { name = "mlx", extras = ["cpu"], marker = "sys_platform == 'linux'", specifier = "==0.30.6" },
-    { name = "mlx-lm", git = "https://github.com/rltakashige/mlx-lm?branch=leo%2Feval-left-padding-in-batched-rotation" },
+    { name = "mlx-lm", git = "https://github.com/rltakashige/mlx-lm?branch=leo%2Ffix-deepseek-v32-indexer" },
     { name = "msgspec", specifier = ">=0.19.0" },
     { name = "openai-harmony", specifier = ">=0.0.8" },
     { name = "psutil", specifier = ">=7.0.0" },
@@ -1342,7 +1354,7 @@ wheels = [
 
 [[package]]
 name = "mflux"
-version = "0.16.9"
+version = "0.17.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -1351,12 +1363,13 @@ dependencies = [
     { name = "huggingface-hub", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "matplotlib", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "mlx", version = "0.30.6", source = { registry = "https://pypi.org/simple" }, extra = ["cuda13"], marker = "sys_platform == 'linux'" },
-    { name = "mlx", version = "0.30.7.dev20260303+257d5692", source = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks#257d5692fc7af6bba3b8afaeb63c549b7d1e43d5" }, marker = "sys_platform == 'darwin'" },
+    { name = "mlx", version = "0.31.2.dev20260324+e5e64331", source = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks#e5e64331830d9b04ae9082b843073f9c1fa7705e" }, marker = "sys_platform == 'darwin'" },
     { name = "numpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "opencv-python", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "piexif", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pillow", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
+    { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "regex", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "safetensors", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -1369,9 +1382,9 @@ dependencies = [
     { name = "twine", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "urllib3", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/e3/dc/73dfccae395df22623fbb6f748657700fd9372be22904a0dafb825dc42d2/mflux-0.16.9.tar.gz", hash = "sha256:ff35e9386b5f026a6b97c786b3426e5341105de3356e55ec1b0c5951ff0ac8c8", size = 764296, upload-time = "2026-03-07T11:54:45.031Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/1b/75/65f791f54c9531b524813d18ff87a0d34866ef220a8fe2ad637437f3cb54/mflux-0.17.2.tar.gz", hash = "sha256:52dee2d27cf438a84648e5c1861b92ceb63a9ac06823d14452a78646a1d30ee7", size = 779264, upload-time = "2026-03-23T13:08:18.377Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/47/69/091df2d0f4a2677d5e2a8b326d568bd884d7ca37f15d3ff78e574aca2271/mflux-0.16.9-py3-none-any.whl", hash = "sha256:988e925653a024e81b46401fd5a2e423dd283428ba64949af4d9257e7df8ddd9", size = 1018804, upload-time = "2026-03-07T11:54:43.47Z" },
+    { url = "https://files.pythonhosted.org/packages/68/02/f94eca4e77b7d12685060461eb793cbc8c00e96cc7fe0ce376374201aed2/mflux-0.17.2-py3-none-any.whl", hash = "sha256:be1642b04847413c0a8ed1dae82ce1ca023e155b057d82a8301eca9c3fe08339", size = 1037451, upload-time = "2026-03-23T13:08:16.747Z" },
 ]
 
 [[package]]
@@ -1399,8 +1412,8 @@ cuda13 = [
 
 [[package]]
 name = "mlx"
-version = "0.30.7.dev20260303+257d5692"
-source = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks#257d5692fc7af6bba3b8afaeb63c549b7d1e43d5" }
+version = "0.31.2.dev20260324+e5e64331"
+source = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks#e5e64331830d9b04ae9082b843073f9c1fa7705e" }
 resolution-markers = [
     "python_full_version >= '3.14' and sys_platform == 'darwin'",
     "python_full_version < '3.14' and sys_platform == 'darwin'",
@@ -1432,11 +1445,11 @@ wheels = [
 
 [[package]]
 name = "mlx-lm"
-version = "0.31.0"
-source = { git = "https://github.com/rltakashige/mlx-lm?branch=leo%2Feval-left-padding-in-batched-rotation#5e0c484cfb5c68d281a71409927ee1bb75adaae2" }
+version = "0.31.2"
+source = { git = "https://github.com/rltakashige/mlx-lm?branch=leo%2Ffix-deepseek-v32-indexer#d388ff77858fec3b5d2e3b1d9502a7e2878b8109" }
 dependencies = [
     { name = "jinja2", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
-    { name = "mlx", version = "0.30.7.dev20260303+257d5692", source = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks#257d5692fc7af6bba3b8afaeb63c549b7d1e43d5" }, marker = "sys_platform == 'darwin'" },
+    { name = "mlx", version = "0.31.2.dev20260324+e5e64331", source = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks#e5e64331830d9b04ae9082b843073f9c1fa7705e" }, marker = "sys_platform == 'darwin'" },
     { name = "numpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
     { name = "pyyaml", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" },
@@ -2091,15 +2104,14 @@ wheels = [
 
 [[package]]
 name = "protobuf"
-version = "6.33.3"
+version = "5.29.6"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/cc/5c/f912bdebdd4af4160da6a2c2b1b3aaa1b8c578d0243ba8f694f93c7095f0/protobuf-6.33.3.tar.gz", hash = "sha256:c8794debeb402963fddff41a595e1f649bcd76616ba56c835645cab4539e810e", size = 444318, upload-time = "2026-01-09T23:05:02.79Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/7e/57/394a763c103e0edf87f0938dafcd918d53b4c011dfc5c8ae80f3b0452dbb/protobuf-5.29.6.tar.gz", hash = "sha256:da9ee6a5424b6b30fd5e45c5ea663aef540ca95f9ad99d1e887e819cdf9b8723", size = 425623, upload-time = "2026-02-04T22:54:40.584Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ec/5d/0ef28dded98973a26443a6a7bc49bff6206be8c57dc1d1e28e6c1147b879/protobuf-6.33.3-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:648b7b0144222eb06cf529a3d7b01333c5f30b4196773b682d388f04db373759", size = 427594, upload-time = "2026-01-09T23:04:53.358Z" },
-    { url = "https://files.pythonhosted.org/packages/c5/46/551c69b6ff1957bd703654342bfb776bb97db400bc80afc56fbb64e7c11d/protobuf-6.33.3-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:08a6ca12f60ba99097dd3625ef4275280f99c9037990e47ce9368826b159b890", size = 324469, upload-time = "2026-01-09T23:04:54.332Z" },
-    { url = "https://files.pythonhosted.org/packages/ca/6d/ade1cca06c64a421ee9745e082671465ead28164c809efaf2c15bc93f9a0/protobuf-6.33.3-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:642fce7187526c98683c79a3ad68e5d646a5ef5eb004582fe123fc9a33a9456b", size = 339242, upload-time = "2026-01-09T23:04:55.347Z" },
-    { url = "https://files.pythonhosted.org/packages/38/8c/6522b8e543ece46f645911c3cebe361d8460134c0fee02ddcf70ebf32999/protobuf-6.33.3-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:6fa9b5f4baa12257542273e5e6f3c3d3867b30bc2770c14ad9ac8315264bf986", size = 323298, upload-time = "2026-01-09T23:04:56.866Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/b9/067b8a843569d5605ba6f7c039b9319720a974f82216cd623e13186d3078/protobuf-6.33.3-py3-none-any.whl", hash = "sha256:c2bf221076b0d463551efa2e1319f08d4cffcc5f0d864614ccd3d0e77a637794", size = 170518, upload-time = "2026-01-09T23:05:01.227Z" },
+    { url = "https://files.pythonhosted.org/packages/06/db/49b05966fd208ae3f44dcd33837b6243b4915c57561d730a43f881f24dea/protobuf-5.29.6-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:b5a169e664b4057183a34bdc424540e86eea47560f3c123a0d64de4e137f9269", size = 418619, upload-time = "2026-02-04T22:54:30.266Z" },
+    { url = "https://files.pythonhosted.org/packages/b7/d7/48cbf6b0c3c39761e47a99cb483405f0fde2be22cf00d71ef316ce52b458/protobuf-5.29.6-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:a8866b2cff111f0f863c1b3b9e7572dc7eaea23a7fae27f6fc613304046483e6", size = 320284, upload-time = "2026-02-04T22:54:31.782Z" },
+    { url = "https://files.pythonhosted.org/packages/e3/dd/cadd6ec43069247d91f6345fa7a0d2858bef6af366dbd7ba8f05d2c77d3b/protobuf-5.29.6-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:e3387f44798ac1106af0233c04fb8abf543772ff241169946f698b3a9a3d3ab9", size = 320478, upload-time = "2026-02-04T22:54:32.909Z" },
+    { url = "https://files.pythonhosted.org/packages/5a/cb/e3065b447186cb70aa65acc70c86baf482d82bf75625bf5a2c4f6919c6a3/protobuf-5.29.6-py3-none-any.whl", hash = "sha256:6b9edb641441b2da9fa8f428760fc136a49cf97a52076010cf22a2ff73438a86", size = 173126, upload-time = "2026-02-04T22:54:39.462Z" },
 ]
 
 [[package]]