fix: only shutdown runners that error unrecoverably

previously we would shutdown a runner that had previously failed,
leading to loops of create -> shutdown -> create.

now when a runner critically errors, we set its status to Shutdown, and
delete it. a failed runner is expected

.mlx_typings/mlx/core/__init__.pyi

@@ -2396,7 +2396,7 @@ def degrees(a: array, /, *, stream: Stream | Device | None = ...) -> array:
         array: The angles in degrees.
     """
 
-def depends(inputs: array | Sequence[array], dependencies: array | Sequence[array]):
+def depends[T](inputs: T, dependencies: array | Sequence[array]) -> T:
     """
     Insert dependencies between arrays in the graph. The outputs are
     identical to ``inputs`` but with dependencies on ``dependencies``.

.mlx_typings/mlx/nn/__init__.pyi

@@ -1,9 +1,5 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from layers import *
-from utils import *
+from .layers import *
+from .utils import *
 
 from . import init as init
 from . import losses as losses

.mlx_typings/mlx/nn/layers/__init__.pyi

@@ -1,20 +1,16 @@
-"""
-This type stub file was generated by pyright.
-"""
-
-from activations import *
-from base import *
-from containers import *
-from convolution import *
-from convolution_transpose import *
-from distributed import *
-from dropout import *
-from embedding import *
-from linear import *
-from normalization import *
-from pooling import *
-from positional_encoding import *
-from quantized import *
-from recurrent import *
-from transformer import *
-from upsample import *
+from .activations import *
+from .base import *
+from .containers import *
+from .convolution import *
+from .convolution_transpose import *
+from .distributed import *
+from .dropout import *
+from .embedding import *
+from .linear import *
+from .normalization import *
+from .pooling import *
+from .positional_encoding import *
+from .quantized import *
+from .recurrent import *
+from .transformer import *
+from .upsample import *

.mlx_typings/mlx/nn/layers/base.pyi

@@ -53,7 +53,7 @@ class Module(dict):
         mx.eval(model.parameters())
     """
 
-    __call__: Callable
+    def __call__(self, *args: Any, **kwargs: Any) -> mx.array: ...
     def __init__(self) -> None:
         """Should be called by the subclasses of ``Module``."""
 

.mlx_typings/mlx/nn/layers/convolution.pyi

@@ -32,6 +32,7 @@ class Conv1d(Module):
     """
 
     weight: mx.array
+    bias: mx.array | None
     groups: int
     def __init__(
         self,

.mlx_typings/mlx/nn/layers/linear.pyi

@@ -40,6 +40,10 @@ class Linear(Module):
         bias (bool, optional): If set to ``False`` then the layer will
           not use a bias. Default is ``True``.
     """
+
+    weight: mx.array
+    bias: mx.array | None
+
     def __init__(self, input_dims: int, output_dims: int, bias: bool = ...) -> None: ...
     def __call__(self, x: mx.array) -> mx.array: ...
     def to_quantized(

.mlx_typings/mlx/nn/layers/normalization.pyi

@@ -88,6 +88,9 @@ class RMSNorm(Module):
         dims (int): The feature dimension of the input to normalize over
         eps (float): A small additive constant for numerical stability
     """
+
+    weight: mx.array
+
     def __init__(self, dims: int, eps: float = ...) -> None: ...
     def __call__(self, x) -> mx.array: ...
 

.mlx_typings/mlx_lm/generate.pyi

@@ -30,7 +30,7 @@ def str2bool(string):  # -> bool:
 def setup_arg_parser():  # -> ArgumentParser:
     """Set up and return the argument parser."""
 
-generation_stream = ...
+generation_stream: mx.Stream
 
 @contextlib.contextmanager
 def wired_limit(
@@ -266,12 +266,12 @@ def _merge_caches(caches: Any) -> List[Any]: ...
 class Batch:
     uids: List[int]
     y: mx.array
-    logprobs: mx.array
+    logprobs: List[mx.array] | mx.array
     max_tokens: List[int]
     num_tokens: List[int]
     cache: List[Any]
-    samplers: List[Any]
-    logits_processors: List[Any]
+    samplers: List[Callable[[mx.array], mx.array] | None]
+    logits_processors: List[List[Callable[[mx.array, mx.array], mx.array]]]
     tokens: List[mx.array]
     def __len__(self) -> int: ...
     def filter(self, keep_idx: List[int]) -> None: ...
@@ -279,13 +279,18 @@ class Batch:
     def extract_cache(self, idx: int) -> List[Any]: ...
 
 class BatchGenerator:
-    model: Any
+    model: nn.Module
+    sampler: Callable[[mx.array], mx.array]
+    stop_tokens: set[int]
     max_kv_size: Optional[int]
     prefill_step_size: int
+    completion_batch_size: int
+    prefill_batch_size: int
     unprocessed_prompts: List[Any]
     active_batch: Optional[Batch]
     prompt_progress_callback: Callable[[List[Tuple[int, int, int]]], None]
     _stats: BatchStats
+    _next_count: int
 
     @dataclass
     class Response:

.mlx_typings/mlx_lm/models/cache.pyi

@@ -88,8 +88,8 @@ def create_attention_mask(
 ) -> array | Literal["causal"] | None: ...
 
 class _BaseCache(Cache):
-    keys: mx.array
-    values: mx.array
+    keys: mx.array | None
+    values: mx.array | None
     offset: int
     @property
     def state(self) -> tuple[mx.array | None, mx.array | None]: ...
@@ -268,29 +268,14 @@ class CacheList(_BaseCache):
         """
 
 class BatchKVCache(_BaseCache):
-    step = ...
-    def __init__(self, left_padding: List[int]) -> None:
-        """
-        The BatchKV cache expects inputs to be left-padded.
-
-        E.g. the following prompts:
-
-            [1, 3, 5]
-            [7]
-            [2, 6, 8, 9]
-
-        Should be padded like so:
-
-            [0, 1, 3, 5]
-            [0, 0, 0, 7]
-            [2, 6, 8, 9]
-
-        And ``left_padding`` specifies the amount of padding for each.
-        In this case, ``left_padding = [1, 3, 0]``.
-        """
-
-    def update_and_fetch(self, keys, values):  # -> tuple[array | Any, array | Any]:
-        ...
+    step: int
+    keys: array | None
+    values: array | None
+    offset: array
+    left_padding: array
+    _idx: int
+    def __init__(self, left_padding: List[int]) -> None: ...
+    def update_and_fetch(self, keys: array, values: array) -> tuple[array, array]: ...
     @property
     def state(
         self,
@@ -316,12 +301,21 @@ class BatchKVCache(_BaseCache):
         """
 
 class BatchRotatingKVCache(_BaseCache):
-    step = ...
-    def __init__(self, max_size, left_padding: List[int]) -> None: ...
-    def update_and_fetch(
-        self, keys, values
-    ):  # -> tuple[array | Any, array | Any] | tuple[array | Any, array | Any | None]:
-        ...
+    step: int
+    keys: array | None
+    values: array | None
+    offset: array
+    left_padding: array
+    max_size: int
+    _idx: int
+    _offset: int
+    rotated: bool
+    _lengths: array | None
+    def __init__(self, max_size: int, left_padding: List[int]) -> None: ...
+    def _trim(self, trim_size: int, v: array, append: array | None = ...) -> array: ...
+    def _update_in_place(self, keys: array, values: array) -> tuple[array, array]: ...
+    def _update_concat(self, keys: array, values: array) -> tuple[array, array]: ...
+    def update_and_fetch(self, keys: array, values: array) -> tuple[array, array]: ...
     @property
     def state(
         self,

.mlx_typings/mlx_lm/models/gated_delta.pyi

@@ -0,0 +1,35 @@
+from typing import Optional
+
+import mlx.core as mx
+
+def compute_g(A_log: mx.array, a: mx.array, dt_bias: mx.array) -> mx.array: ...
+def gated_delta_update(
+    q: mx.array,
+    k: mx.array,
+    v: mx.array,
+    a: mx.array,
+    b: mx.array,
+    A_log: mx.array,
+    dt_bias: mx.array,
+    state: Optional[mx.array] = ...,
+    mask: Optional[mx.array] = ...,
+    use_kernel: bool = ...,
+) -> tuple[mx.array, mx.array]: ...
+def gated_delta_ops(
+    q: mx.array,
+    k: mx.array,
+    v: mx.array,
+    g: mx.array,
+    beta: mx.array,
+    state: Optional[mx.array] = ...,
+    mask: Optional[mx.array] = ...,
+) -> tuple[mx.array, mx.array]: ...
+def gated_delta_kernel(
+    q: mx.array,
+    k: mx.array,
+    v: mx.array,
+    g: mx.array,
+    beta: mx.array,
+    state: mx.array,
+    mask: Optional[mx.array] = ...,
+) -> tuple[mx.array, mx.array]: ...

.mlx_typings/mlx_lm/models/nemotron_h.pyi

@@ -0,0 +1,154 @@
+from dataclasses import dataclass
+from typing import Any, List, Optional, Tuple
+
+import mlx.core as mx
+import mlx.nn as nn
+
+from .cache import ArraysCache, KVCache
+from .switch_layers import SwitchMLP
+
+@dataclass
+class ModelArgs:
+    model_type: str
+    vocab_size: int
+    hidden_size: int
+    intermediate_size: int
+    num_hidden_layers: int
+    max_position_embeddings: int
+    num_attention_heads: int
+    num_key_value_heads: int
+    attention_bias: bool
+    mamba_num_heads: int
+    mamba_head_dim: int
+    mamba_proj_bias: bool
+    ssm_state_size: int
+    conv_kernel: int
+    n_groups: int
+    mlp_bias: bool
+    layer_norm_epsilon: float
+    use_bias: bool
+    use_conv_bias: bool
+    hybrid_override_pattern: List[str]
+    head_dim: Optional[int]
+    moe_intermediate_size: Optional[int]
+    moe_shared_expert_intermediate_size: Optional[int]
+    n_group: Optional[int]
+    n_routed_experts: Optional[int]
+    n_shared_experts: Optional[int]
+    topk_group: Optional[int]
+    num_experts_per_tok: Optional[int]
+    norm_topk_prob: Optional[bool]
+    routed_scaling_factor: Optional[float]
+    time_step_limit: Optional[Tuple[float, float]]
+    time_step_min: Optional[float]
+    time_step_max: Optional[float]
+
+    @classmethod
+    def from_dict(cls, params: dict[str, Any]) -> ModelArgs: ...
+    def __post_init__(self) -> None: ...
+
+class NemotronHMamba2Mixer(nn.Module):
+    num_heads: int
+    hidden_size: int
+    ssm_state_size: int
+    conv_kernel_size: int
+    intermediate_size: int
+    n_groups: int
+    head_dim: int
+    conv_dim: int
+    conv1d: nn.Conv1d
+    in_proj: nn.Linear
+    dt_bias: mx.array
+    A_log: mx.array
+    D: mx.array
+    norm: nn.RMSNorm
+    heads_per_group: int
+    out_proj: nn.Linear
+
+    def __init__(self, args: ModelArgs) -> None: ...
+    def __call__(
+        self,
+        hidden_states: mx.array,
+        mask: Optional[mx.array],
+        cache: Optional[ArraysCache] = None,
+    ) -> mx.array: ...
+
+class NemotronHAttention(nn.Module):
+    hidden_size: int
+    num_heads: int
+    head_dim: int
+    num_key_value_heads: int
+    scale: float
+    q_proj: nn.Linear
+    k_proj: nn.Linear
+    v_proj: nn.Linear
+    o_proj: nn.Linear
+
+    def __init__(self, args: ModelArgs) -> None: ...
+    def __call__(
+        self,
+        x: mx.array,
+        mask: Optional[mx.array] = None,
+        cache: Optional[KVCache] = None,
+    ) -> mx.array: ...
+
+class NemotronHMLP(nn.Module):
+    up_proj: nn.Linear
+    down_proj: nn.Linear
+
+    def __init__(
+        self, args: ModelArgs, intermediate_size: Optional[int] = None
+    ) -> None: ...
+    def __call__(self, x: mx.array) -> mx.array: ...
+
+class NemotronHMoE(nn.Module):
+    num_experts_per_tok: int
+    switch_mlp: SwitchMLP
+    shared_experts: NemotronHMLP
+
+    def __init__(self, config: ModelArgs) -> None: ...
+    def __call__(self, x: mx.array) -> mx.array: ...
+
+class NemotronHBlock(nn.Module):
+    block_type: str
+    norm: nn.RMSNorm
+    mixer: NemotronHMamba2Mixer | NemotronHAttention | NemotronHMLP | NemotronHMoE
+
+    def __init__(self, args: ModelArgs, block_type: str) -> None: ...
+    def __call__(
+        self,
+        x: mx.array,
+        mask: Optional[mx.array] = None,
+        cache: Optional[Any] = None,
+    ) -> mx.array: ...
+
+class NemotronHModel(nn.Module):
+    embeddings: nn.Embedding
+    layers: list[NemotronHBlock]
+    norm_f: nn.RMSNorm
+    fa_idx: int
+    ssm_idx: int
+
+    def __init__(self, args: ModelArgs) -> None: ...
+    def __call__(
+        self,
+        inputs: mx.array,
+        cache: Optional[Any] = None,
+    ) -> mx.array: ...
+
+class Model(nn.Module):
+    args: ModelArgs
+    backbone: NemotronHModel
+    lm_head: nn.Linear
+    model_type: str
+
+    def __init__(self, args: ModelArgs) -> None: ...
+    def __call__(
+        self,
+        inputs: mx.array,
+        cache: Optional[Any] = None,
+    ) -> mx.array: ...
+    @property
+    def layers(self) -> list[NemotronHBlock]: ...
+    def make_cache(self) -> list[ArraysCache | KVCache]: ...
+    def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...

.mlx_typings/mlx_lm/models/qwen3_next.pyi

@@ -5,6 +5,7 @@ from typing import Any, Optional
 import mlx.core as mx
 import mlx.nn as nn
 
+from .cache import ArraysCache, KVCache
 from .switch_layers import SwitchGLU
 
 class Qwen3NextRMSNormGated(nn.Module):
@@ -99,6 +100,8 @@ class Qwen3NextModel(nn.Module):
     embed_tokens: nn.Embedding
     layers: list[Qwen3NextDecoderLayer]
     norm: nn.RMSNorm
+    ssm_idx: int
+    fa_idx: int
 
     def __init__(self, args: Any) -> None: ...
     def __call__(
@@ -121,3 +124,4 @@ class Model(nn.Module):
     def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
     @property
     def layers(self) -> list[Qwen3NextDecoderLayer]: ...
+    def make_cache(self) -> list[ArraysCache | KVCache]: ...

.mlx_typings/mlx_lm/models/rope_utils.pyi

@@ -0,0 +1,51 @@
+from typing import Any, Optional
+
+import mlx.nn as nn
+
+class YarnRoPE(nn.Module):
+    def __init__(
+        self,
+        dims: int,
+        traditional: bool = ...,
+        max_position_embeddings: int = ...,
+        base: float = ...,
+        scaling_factor: float = ...,
+        original_max_position_embeddings: int = ...,
+        beta_fast: float = ...,
+        beta_slow: float = ...,
+        mscale: float = ...,
+        mscale_all_dim: float = ...,
+    ) -> None: ...
+
+class Llama3RoPE(nn.Module):
+    def __init__(
+        self,
+        dims: int,
+        traditional: bool = ...,
+        max_position_embeddings: int = ...,
+        base: float = ...,
+        scaling_factor: float = ...,
+        original_max_position_embeddings: int = ...,
+        low_freq_factor: float = ...,
+        high_freq_factor: float = ...,
+    ) -> None: ...
+
+class SuScaledRoPE(nn.Module):
+    def __init__(
+        self,
+        dims: int,
+        traditional: bool = ...,
+        max_position_embeddings: int = ...,
+        base: float = ...,
+        short_factor: Any = ...,
+        long_factor: Any = ...,
+        original_max_position_embeddings: int = ...,
+    ) -> None: ...
+
+def initialize_rope(
+    dims: int,
+    base: float = ...,
+    traditional: bool = ...,
+    scaling_config: Optional[dict[str, Any]] = ...,
+    max_position_embeddings: Optional[int] = ...,
+) -> nn.Module: ...

.mlx_typings/mlx_lm/models/switch_layers.pyi

@@ -73,6 +73,9 @@ class SwitchGLU(nn.Module):
     def __call__(self, x, indices) -> mx.array: ...
 
 class SwitchMLP(nn.Module):
+    fc1: SwitchLinear
+    fc2: SwitchLinear
+
     def __init__(
         self,
         input_dims: int,

.mlx_typings/mlx_lm/tokenizer_utils.pyi

@@ -39,11 +39,11 @@ class StreamingDetokenizer:
     """
 
     __slots__ = ...
-    def reset(self): ...
-    def add_token(self, token): ...
-    def finalize(self): ...
+    def reset(self) -> None: ...
+    def add_token(self, token: int) -> None: ...
+    def finalize(self) -> None: ...
     @property
-    def last_segment(self):
+    def last_segment(self) -> str:
         """Return the last segment of readable text since last time this property was accessed."""
 
 class NaiveStreamingDetokenizer(StreamingDetokenizer):

CONTRIBUTING.md

@@ -11,9 +11,18 @@ To run EXO from source:
   ```bash
   brew install uv
   ```
-- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
   ```bash
-  brew install macmon
+  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+  rustup toolchain install nightly
+  ```
+- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+  Use the pinned fork revision used by this repo instead of Homebrew `macmon`.
+  ```bash
+  cargo install --git https://github.com/swiftraccoon/macmon \
+    --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
+    macmon \
+    --force
   ```
 
 ```bash

README.md

@@ -95,11 +95,10 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
   ```
 - [uv](https://github.com/astral-sh/uv) (for Python dependency management)
-- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
 - [node](https://github.com/nodejs/node) (for building the dashboard)
 
   ```bash
-  brew install uv macmon node
+  brew install uv node
   ```
 - [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
 
@@ -107,6 +106,17 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
   rustup toolchain install nightly
   ```
+- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+
+  Install the pinned fork revision used by this repo instead of Homebrew `macmon`.
+  Homebrew `macmon 0.6.1` still crashes on Apple M5.
+
+  ```bash
+  cargo install --git https://github.com/swiftraccoon/macmon \
+    --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
+    macmon \
+    --force
+  ```
 
 Clone the repo, build the dashboard, and run exo:
 

app/EXO/EXO/ExoProcessController.swift

@@ -4,6 +4,7 @@ import Foundation
 
 private let customNamespaceKey = "EXOCustomNamespace"
 private let hfTokenKey = "EXOHFToken"
+private let hfEndpointKey = "EXOHFEndpoint"
 private let enableImageModelsKey = "EXOEnableImageModels"
 private let offlineModeKey = "EXOOfflineMode"
 private let onboardingCompletedKey = "EXOOnboardingCompleted"
@@ -53,6 +54,14 @@ final class ExoProcessController: ObservableObject {
             UserDefaults.standard.set(hfToken, forKey: hfTokenKey)
         }
     }
+    @Published var hfEndpoint: String = {
+        return UserDefaults.standard.string(forKey: hfEndpointKey) ?? ""
+    }()
+    {
+        didSet {
+            UserDefaults.standard.set(hfEndpoint, forKey: hfEndpointKey)
+        }
+    }
     @Published var enableImageModels: Bool = {
         return UserDefaults.standard.bool(forKey: enableImageModelsKey)
     }()
@@ -273,6 +282,9 @@ final class ExoProcessController: ObservableObject {
         if !hfToken.isEmpty {
             environment["HF_TOKEN"] = hfToken
         }
+        if !hfEndpoint.isEmpty {
+            environment["HF_ENDPOINT"] = hfEndpoint
+        }
         if enableImageModels {
             environment["EXO_ENABLE_IMAGE_MODELS"] = "true"
         }

app/EXO/EXO/Views/SettingsView.swift

@@ -12,6 +12,7 @@ struct SettingsView: View {
 
     @State private var pendingNamespace: String = ""
     @State private var pendingHFToken: String = ""
+    @State private var pendingHFEndpoint: String = ""
     @State private var pendingEnableImageModels = false
     @State private var pendingOfflineMode = false
     @State private var needsRestart = false
@@ -42,6 +43,7 @@ struct SettingsView: View {
         .onAppear {
             pendingNamespace = controller.customNamespace
             pendingHFToken = controller.hfToken
+            pendingHFEndpoint = controller.hfEndpoint
             pendingEnableImageModels = controller.enableImageModels
             pendingOfflineMode = controller.offlineMode
             needsRestart = false
@@ -74,6 +76,17 @@ struct SettingsView: View {
                     .foregroundColor(.secondary)
             }
 
+            Section {
+                LabeledContent("HuggingFace Endpoint") {
+                    TextField("default", text: $pendingHFEndpoint)
+                        .textFieldStyle(.roundedBorder)
+                        .frame(width: 200)
+                }
+                Text("Defaults to huggingface.co. Use a mirror (e.g. hf-mirror.com) for China.")
+                    .font(.caption)
+                    .foregroundColor(.secondary)
+            }
+
             Section {
                 Toggle("Offline Mode", isOn: $pendingOfflineMode)
                 Text("Skip internet checks and use only locally available models.")
@@ -454,6 +467,7 @@ struct SettingsView: View {
 
     private var hasGeneralChanges: Bool {
         pendingNamespace != controller.customNamespace || pendingHFToken != controller.hfToken
+            || pendingHFEndpoint != controller.hfEndpoint
             || pendingOfflineMode != controller.offlineMode
     }
 
@@ -464,6 +478,7 @@ struct SettingsView: View {
     private func applyGeneralSettings() {
         controller.customNamespace = pendingNamespace
         controller.hfToken = pendingHFToken
+        controller.hfEndpoint = pendingHFEndpoint
         controller.offlineMode = pendingOfflineMode
         restartIfRunning()
     }

bench/exo_bench.py

@@ -496,20 +496,26 @@ def main() -> int:
                                 all_rows.append(row)
 
                             if batch_results:
-                                agg_gen_tps = sum(
+                                valid_gen_tps = [
                                     x["stats"]["generation_tps"]
                                     for x, _ in batch_results
                                     if x["stats"]["generation_tps"] > 0
+                                ]
+                                per_req_tps = (
+                                    mean(valid_gen_tps) if valid_gen_tps else 0.0
                                 )
+                                agg_gen_tps = per_req_tps * concurrency
                                 logger.info(
                                     f"[concurrent {concurrency}x]  "
                                     f"agg_gen_tps={agg_gen_tps:.2f}  "
+                                    f"per_req_tps={per_req_tps:.2f}  "
                                     f"errors={batch_errors}"
                                 )
 
                     if runs:
                         prompt_tps = mean(x["stats"]["prompt_tps"] for x in runs)
-                        gen_tps = mean(x["stats"]["generation_tps"] for x in runs)
+                        per_req_tps = mean(x["stats"]["generation_tps"] for x in runs)
+                        gen_tps = per_req_tps * concurrency
                         ptok = mean(x["stats"]["prompt_tokens"] for x in runs)
                         gtok = mean(x["stats"]["generation_tokens"] for x in runs)
                         peak = mean(

dashboard/src/lib/components/ChatForm.svelte

@@ -1,9 +1,6 @@
 <script lang="ts">
   import {
     isLoading,
-    sendMessage,
-    generateImage,
-    editImage,
     editingImage,
     clearEditingImage,
     selectedChatModel,
@@ -28,7 +25,7 @@
     modelTasks?: Record<string, string[]>;
     modelCapabilities?: Record<string, string[]>;
     onSend?: () => void;
-    onAutoSend?: (
+    onAutoSend: (
       content: string,
       files?: {
         id: string;
@@ -216,49 +213,10 @@
     uploadedFiles = [];
     resetTextareaHeight();
 
-    // When onAutoSend is provided, the parent controls all send logic
-    // (including launching non-running models before sending)
-    if (onAutoSend) {
-      onAutoSend(content, files);
-      onSend?.();
-      setTimeout(() => textareaRef?.focus(), 10);
-      return;
-    }
-
-    // Use image editing if in edit mode
-    if (isEditMode && currentEditingImage && content) {
-      editImage(content, currentEditingImage.imageDataUrl);
-    }
-    // If user attached an image with an ImageToImage model, use edit endpoint
-    else if (
-      currentModel &&
-      modelSupportsImageEditing(currentModel) &&
-      files.length > 0 &&
-      content
-    ) {
-      // Use the first attached image for editing
-      const imageFile = files[0];
-      if (imageFile.preview) {
-        editImage(content, imageFile.preview);
-      }
-    } else if (
-      currentModel &&
-      modelSupportsTextToImage(currentModel) &&
-      content
-    ) {
-      // Use image generation for text-to-image models
-      generateImage(content);
-    } else {
-      sendMessage(
-        content,
-        files,
-        modelSupportsThinking() ? thinkingEnabled : null,
-      );
-    }
-
+    // Parent controls all send logic (including image routing,
+    // launching non-running models before sending, etc.)
+    onAutoSend(content, files);
     onSend?.();
-
-    // Refocus the textarea after sending
     setTimeout(() => textareaRef?.focus(), 10);
   }
 

dashboard/src/lib/components/FamilyLogos.svelte

@@ -82,6 +82,18 @@
       d="M12.025 1.13c-5.77 0-10.449 4.647-10.449 10.378 0 1.112.178 2.181.503 3.185.064-.222.203-.444.416-.577a.96.96 0 0 1 .524-.15c.293 0 .584.124.84.284.278.173.48.408.71.694.226.282.458.611.684.951v-.014c.017-.324.106-.622.264-.874s.403-.487.762-.543c.3-.047.596.06.787.203s.31.313.4.467c.15.257.212.468.233.542.01.026.653 1.552 1.657 2.54.616.605 1.01 1.223 1.082 1.912.055.537-.096 1.059-.38 1.572.637.121 1.294.187 1.967.187.657 0 1.298-.063 1.921-.178-.287-.517-.44-1.041-.384-1.581.07-.69.465-1.307 1.081-1.913 1.004-.987 1.647-2.513 1.657-2.539.021-.074.083-.285.233-.542.09-.154.208-.323.4-.467a1.08 1.08 0 0 1 .787-.203c.359.056.604.29.762.543s.247.55.265.874v.015c.225-.34.457-.67.683-.952.23-.286.432-.52.71-.694.257-.16.547-.284.84-.285a.97.97 0 0 1 .524.151c.228.143.373.388.43.625l.006.04a10.3 10.3 0 0 0 .534-3.273c0-5.731-4.678-10.378-10.449-10.378M8.327 6.583a1.5 1.5 0 0 1 .713.174 1.487 1.487 0 0 1 .617 2.013c-.183.343-.762-.214-1.102-.094-.38.134-.532.914-.917.71a1.487 1.487 0 0 1 .69-2.803m7.486 0a1.487 1.487 0 0 1 .689 2.803c-.385.204-.536-.576-.916-.71-.34-.12-.92.437-1.103.094a1.487 1.487 0 0 1 .617-2.013 1.5 1.5 0 0 1 .713-.174m-10.68 1.55a.96.96 0 1 1 0 1.921.96.96 0 0 1 0-1.92m13.838 0a.96.96 0 1 1 0 1.92.96.96 0 0 1 0-1.92M8.489 11.458c.588.01 1.965 1.157 3.572 1.164 1.607-.007 2.984-1.155 3.572-1.164.196-.003.305.12.305.454 0 .886-.424 2.328-1.563 3.202-.22-.756-1.396-1.366-1.63-1.32q-.011.001-.02.006l-.044.026-.01.008-.03.024q-.018.017-.035.036l-.032.04a1 1 0 0 0-.058.09l-.014.025q-.049.088-.11.19a1 1 0 0 1-.083.116 1.2 1.2 0 0 1-.173.18q-.035.029-.075.058a1.3 1.3 0 0 1-.251-.243 1 1 0 0 1-.076-.107c-.124-.193-.177-.363-.337-.444-.034-.016-.104-.008-.2.022q-.094.03-.216.087-.06.028-.125.063l-.13.074q-.067.04-.136.086a3 3 0 0 0-.135.096 3 3 0 0 0-.26.219 2 2 0 0 0-.12.121 2 2 0 0 0-.106.128l-.002.002a2 2 0 0 0-.09.132l-.001.001a1.2 1.2 0 0 0-.105.212q-.013.036-.024.073c-1.139-.875-1.563-2.317-1.563-3.203 0-.334.109-.457.305-.454m.836 10.354c.824-1.19.766-2.082-.365-3.194-1.13-1.112-1.789-2.738-1.789-2.738s-.246-.945-.806-.858-.97 1.499.202 2.362c1.173.864-.233 1.45-.685.64-.45-.812-1.683-2.896-2.322-3.295s-1.089-.175-.938.647 2.822 2.813 2.562 3.244-1.176-.506-1.176-.506-2.866-2.567-3.49-1.898.473 1.23 2.037 2.16c1.564.932 1.686 1.178 1.464 1.53s-3.675-2.511-4-1.297c-.323 1.214 3.524 1.567 3.287 2.405-.238.839-2.71-1.587-3.216-.642-.506.946 3.49 2.056 3.522 2.064 1.29.33 4.568 1.028 5.713-.624m5.349 0c-.824-1.19-.766-2.082.365-3.194 1.13-1.112 1.789-2.738 1.789-2.738s.246-.945.806-.858.97 1.499-.202 2.362c-1.173.864.233 1.45.685.64.451-.812 1.683-2.896 2.322-3.295s1.089-.175.938.647-2.822 2.813-2.562 3.244 1.176-.506 1.176-.506 2.866-2.567 3.49-1.898-.473 1.23-2.037 2.16c-1.564.932-1.686 1.178-1.464 1.53s3.675-2.511 4-1.297c.323 1.214-3.524 1.567-3.287 2.405.238.839 2.71-1.587 3.216-.642.506.946-3.49 2.056-3.522 2.064-1.29.33-4.568 1.028-5.713-.624"
     />
   </svg>
+{:else if family === "step"}
+  <svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
+    <path
+      d="M22.012 0h1.032v.927H24v.968h-.956V3.78h-1.032V1.896h-1.878v-.97h1.878V0zM2.6 12.371V1.87h.969v10.502h-.97zm10.423.66h10.95v.918h-6.208v9.579h-4.742V13.03zM5.629 3.333v12.356H0v4.51h10.386V8L20.859 8l-.003-4.668-15.227.001z"
+    />
+  </svg>
+{:else if family === "nemotron"}
+  <svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
+    <path
+      d="M8.948 8.798v-1.43a6.7 6.7 0 0 1 .424-.018c3.922-.124 6.493 3.374 6.493 3.374s-2.774 3.851-5.75 3.851c-.398 0-.787-.062-1.158-.185v-4.346c1.528.185 1.837.857 2.747 2.385l2.04-1.714s-1.492-1.952-4-1.952a6.016 6.016 0 0 0-.796.035m0-4.735v2.138l.424-.027c5.45-.185 9.01 4.47 9.01 4.47s-4.08 4.964-8.33 4.964c-.37 0-.733-.035-1.095-.097v1.325c.3.035.61.062.91.062 3.957 0 6.82-2.023 9.593-4.408.459.371 2.34 1.263 2.73 1.652-2.633 2.208-8.772 3.984-12.253 3.984-.335 0-.653-.018-.971-.053v1.864H24V4.063zm0 10.326v1.131c-3.657-.654-4.673-4.46-4.673-4.46s1.758-1.944 4.673-2.262v1.237H8.94c-1.528-.186-2.73 1.245-2.73 1.245s.68 2.412 2.739 3.11M2.456 10.9s2.164-3.197 6.5-3.533V6.201C4.153 6.59 0 10.653 0 10.653s2.35 6.802 8.948 7.42v-1.237c-4.84-.6-6.492-5.936-6.492-5.936z"
+    />
+  </svg>
 {:else}
   <svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
     <path

dashboard/src/lib/components/FamilySidebar.svelte

@@ -31,6 +31,7 @@
     kimi: "Kimi",
     flux: "FLUX",
     "qwen-image": "Qwen Img",
+    nemotron: "NVIDIA",
   };
 
   function getFamilyName(family: string): string {

dashboard/src/lib/components/ModelCard.svelte

@@ -16,7 +16,9 @@
       perNode?: Array<{
         nodeId: string;
         nodeName: string;
-        progress: DownloadProgress;
+        status: "completed" | "partial" | "pending" | "downloading";
+        percentage: number;
+        progress: DownloadProgress | null;
       }>;
     } | null;
     nodes?: Record<string, NodeInfo>;
@@ -145,10 +147,7 @@
     return `${s}s`;
   }
 
-  const isDownloading = $derived(downloadStatus?.isDownloading ?? false);
-  const progress = $derived(downloadStatus?.progress);
-  const percentage = $derived(progress?.percentage ?? 0);
-  let expandedNodes = $state<Set<string>>(new Set());
+  const perNode = $derived(downloadStatus?.perNode ?? []);
 
   function toggleNodeDetails(nodeId: string): void {
     const next = new Set(expandedNodes);
@@ -587,23 +586,49 @@
       </span>
     </div>
 
-    <!-- Download Status -->
-    {#if isDownloading && progress}
+    <!-- Download Status (per-node) -->
+    {#if perNode.length > 0}
       <div class="mb-2 space-y-1">
-        <div class="flex items-center justify-between text-xs font-mono">
-          <span class="text-blue-400 tracking-wider uppercase">Downloading</span
-          >
-          <span class="text-white/60"
-            >{percentage.toFixed(1)}% &middot; {formatSpeed(progress.speed)}
-            &middot; {formatEta(progress.etaMs)}</span
-          >
-        </div>
-        <div class="h-1 bg-exo-medium-gray/30 rounded overflow-hidden">
-          <div
-            class="h-full bg-blue-500/70 transition-all duration-300"
-            style="width: {percentage}%"
-          ></div>
+        <div
+          class="text-[10px] font-mono text-white/20 tracking-widest uppercase"
+        >
+          Download progress
         </div>
+        {#each perNode as node}
+          <div class="flex items-center gap-2 text-xs font-mono">
+            <span class="text-white/40 w-20 truncate" title={node.nodeId}
+              >{node.nodeName}</span
+            >
+            <div
+              class="flex-1 h-1 bg-exo-medium-gray/30 rounded overflow-hidden"
+            >
+              <div
+                class="h-full transition-all duration-300 {node.status ===
+                'downloading'
+                  ? 'bg-blue-500/70'
+                  : node.status === 'completed'
+                    ? 'bg-exo-yellow/40'
+                    : 'bg-white/20'}"
+                style="width: {node.percentage}%"
+              ></div>
+            </div>
+            <span
+              class="text-right {node.status === 'completed'
+                ? 'text-exo-yellow/60'
+                : node.status === 'downloading'
+                  ? 'text-blue-400/60'
+                  : 'text-white/30'}"
+            >
+              {#if node.status === "downloading" && node.progress}
+                {Math.round(node.percentage)}% {formatSpeed(
+                  node.progress.speed,
+                )}
+              {:else}
+                {node.percentage > 0 ? `${Math.round(node.percentage)}%` : "0%"}
+              {/if}
+            </span>
+          </div>
+        {/each}
       </div>
     {/if}
 
@@ -662,15 +687,7 @@
             {@const allConnections =
               isDebugMode && usedNodes.length > 1
                 ? (() => {
-                    const conns: Array<{
-                      ip: string;
-                      iface: string | null;
-                      from: string;
-                      to: string;
-                      midX: number;
-                      midY: number;
-                      arrow: string;
-                    }> = [];
+                    const conns: Array = [];
                     for (let i = 0; i < usedNodes.length; i++) {
                       for (let j = i + 1; j < usedNodes.length; j++) {
                         const n1 = usedNodes[i];
@@ -682,7 +699,12 @@
                           const toPos = nodePositions[c.to];
                           const arrow =
                             fromPos && toPos ? getArrow(fromPos, toPos) : "→";
-                          conns.push({ ...c, midX, midY, arrow });
+                          conns.push({
+                            ...c,
+                            midX,
+                            midY,
+                            arrow,
+                          });
                         }
                       }
                     }

dashboard/src/lib/components/ModelPickerModal.svelte

@@ -459,6 +459,7 @@
       "llama",
       "flux",
       "qwen-image",
+      "nemotron",
     ];
     return Array.from(families).sort((a, b) => {
       const aIdx = familyOrder.indexOf(a);

dashboard/src/lib/stores/app.svelte.ts

@@ -1577,6 +1577,7 @@ class AppStore {
     // Remove messages after user message (including the user message for image requests
     // since generateImage/editImage will re-add it)
     this.messages = this.messages.slice(0, lastUserIndex);
+    this.updateActiveConversation();
 
     switch (requestType) {
       case "image-generation":
@@ -1792,6 +1793,14 @@ class AppStore {
             this.persistConversation(targetConversationId);
           }
         },
+        {
+          generation_stats: (data) => {
+            const stats = data as { generation_tps: number };
+            if (stats.generation_tps > 0) {
+              this.tps = stats.generation_tps;
+            }
+          },
+        },
       );
 
       // Final update
@@ -1989,6 +1998,14 @@ class AppStore {
             this.persistConversation(targetConversationId);
           }
         },
+        {
+          generation_stats: (data) => {
+            const stats = data as { generation_tps: number };
+            if (stats.generation_tps > 0) {
+              this.tps = stats.generation_tps;
+            }
+          },
+        },
       );
 
       // Final cleanup of the message (if conversation still exists)
@@ -2396,7 +2413,7 @@ class AppStore {
 
       let streamedContent = "";
       let streamedThinking = "";
-
+      let serverTpsReceived = false;
       interface ChatCompletionChunk {
         choices?: Array<{
           delta?: { content?: string; reasoning_content?: string };
@@ -2461,7 +2478,6 @@ class AppStore {
             tokenCount += 1;
             this.totalTokens = tokenCount;
 
-            // Update real-time TPS during streaming
             if (firstTokenTime !== null && tokenCount > 1) {
               const elapsed = performance.now() - firstTokenTime;
               this.tps = (tokenCount / elapsed) * 1000;
@@ -2512,16 +2528,24 @@ class AppStore {
               startedAt: this.prefillProgress?.startedAt ?? performance.now(),
             };
           },
+          generation_stats: (data) => {
+            const stats = data as { generation_tps: number };
+
+            if (stats.generation_tps > 0) {
+              this.tps = stats.generation_tps;
+              serverTpsReceived = true;
+            }
+          },
         },
       );
 
       // Clear prefill progress after stream ends
       this.prefillProgress = null;
 
-      // Calculate final TPS
-      if (firstTokenTime !== null && tokenCount > 1) {
+      // Use server-side TPS if available, otherwise fall back to client-side
+      if (!serverTpsReceived && firstTokenTime !== null && tokenCount > 1) {
         const totalGenerationTime = performance.now() - firstTokenTime;
-        this.tps = (tokenCount / totalGenerationTime) * 1000; // tokens per second
+        this.tps = (tokenCount / totalGenerationTime) * 1000;
       }
 
       // Final cleanup of the message (if conversation still exists)

dashboard/src/routes/+page.svelte

@@ -42,6 +42,10 @@
     setSelectedChatModel,
     selectedChatModel,
     sendMessage,
+    thinkingEnabled,
+    generateImage,
+    editImage,
+    editingImage,
     messages,
     debugMode,
     toggleDebugMode,
@@ -834,6 +838,52 @@
     if (!model?.tasks) return false;
     return model.tasks.includes("ImageToImage");
   }
+
+  // Route a message to the correct endpoint based on model capabilities.
+  // Image models go to generateImage/editImage; text models go to sendMessage.
+  function routeMessage(
+    content: string,
+    files?: {
+      id: string;
+      name: string;
+      type: string;
+      textContent?: string;
+      preview?: string;
+    }[],
+  ) {
+    const model = selectedChatModel();
+    if (!model) {
+      sendMessage(content, files, thinkingEnabled());
+      return;
+    }
+
+    const currentEditImage = editingImage();
+
+    // Image editing mode (explicit edit or attached image with ImageToImage model)
+    if (currentEditImage && content && modelSupportsImageEditing(model)) {
+      editImage(content, currentEditImage.imageDataUrl);
+      return;
+    }
+    if (
+      modelSupportsImageEditing(model) &&
+      files?.length &&
+      files[0].preview &&
+      content
+    ) {
+      editImage(content, files[0].preview);
+      return;
+    }
+
+    // Text-to-image generation
+    if (modelSupportsImageGeneration(model) && content) {
+      generateImage(content);
+      return;
+    }
+
+    // Default: text chat
+    sendMessage(content, files, thinkingEnabled());
+  }
+
   let selectedSharding = $state<"Pipeline" | "Tensor">("Pipeline");
   type InstanceMeta = "MlxRing" | "MlxJaccl";
 
@@ -1486,34 +1536,44 @@
   }
 
   // Helper to get download status for a model (checks all downloads for matching model ID)
-  function getModelDownloadStatus(modelId: string): {
+  type NodeDownloadStatus = {
+    nodeId: string;
+    nodeName: string;
+    status: "completed" | "partial" | "pending" | "downloading";
+    percentage: number;
+    progress: DownloadProgress | null;
+  };
+
+  // Shared helper: collect per-node download status for a model across a set of nodes.
+  // Handles deduplication, entry parsing, and aggregation in one place.
+  function collectDownloadStatus(
+    modelId: string,
+    nodeIds?: string[],
+  ): {
     isDownloading: boolean;
     progress: DownloadProgress | null;
-    perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }>;
+    perNode: NodeDownloadStatus[];
+    failedError: string | null;
   } {
+    const empty = {
+      isDownloading: false,
+      progress: null,
+      perNode: [] as NodeDownloadStatus[],
+      failedError: null,
+    };
+
     if (!downloadsData || Object.keys(downloadsData).length === 0) {
-      return { isDownloading: false, progress: null, perNode: [] };
+      return empty;
     }
 
-    let totalBytes = 0;
-    let downloadedBytes = 0;
-    let totalSpeed = 0;
-    let completedFiles = 0;
-    let totalFiles = 0;
-    let isDownloading = false;
-    const allFiles: DownloadProgress["files"] = [];
-    const perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }> = [];
+    // Deduplicate by nodeId — a node can have multiple entries for the same model
+    // (e.g. PipelineShardMetadata + TensorShardMetadata). Keep the last entry,
+    // which is the most recently applied event.
+    const perNodeMap = new Map<string, NodeDownloadStatus>();
 
-    // Check all nodes for downloads matching this model
+    const nodeIdSet = nodeIds ? new Set(nodeIds) : null;
     for (const [nodeId, nodeDownloads] of Object.entries(downloadsData)) {
+      if (nodeIdSet && !nodeIdSet.has(nodeId)) continue;
       if (!Array.isArray(nodeDownloads)) continue;
 
       for (const downloadWrapped of nodeDownloads) {
@@ -1526,46 +1586,118 @@
         const downloadPayload = (downloadWrapped as Record<string, unknown>)[
           downloadKind
         ] as Record<string, unknown>;
-
-        if (downloadKind !== "DownloadOngoing") continue;
         if (!downloadPayload) continue;
 
         const downloadModelId = extractModelIdFromDownload(downloadPayload);
+        if (!downloadModelId || downloadModelId !== modelId) continue;
+
+        // DownloadFailed — return with any data collected so far
+        if (downloadKind === "DownloadFailed") {
+          return {
+            isDownloading: false,
+            progress: null,
+            perNode: Array.from(perNodeMap.values()),
+            failedError:
+              (downloadPayload.errorMessage as string) ||
+              (downloadPayload.error_message as string) ||
+              "Download failed",
+          };
+        }
 
-        // Match if the model ID contains or equals the requested model
-        // (handles cases like "mlx-community/Meta-Llama..." matching)
         if (
-          !downloadModelId ||
-          !downloadModelId.includes(modelId.split("/").pop() || modelId)
-        ) {
-          // Try exact match or partial match
-          if (downloadModelId !== modelId) continue;
+          downloadKind !== "DownloadOngoing" &&
+          downloadKind !== "DownloadPending" &&
+          downloadKind !== "DownloadCompleted"
+        )
+          continue;
+
+        const nodeName =
+          data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
+
+        if (downloadKind === "DownloadCompleted") {
+          perNodeMap.set(nodeId, {
+            nodeId,
+            nodeName,
+            status: "completed",
+            percentage: 100,
+            progress: null,
+          });
+          continue;
         }
 
-        isDownloading = true;
+        if (downloadKind === "DownloadPending") {
+          const pendingDownloaded = getBytes(
+            downloadPayload.downloaded ??
+              downloadPayload.downloaded_bytes ??
+              downloadPayload.downloadedBytes,
+          );
+          const pendingTotal = getBytes(
+            downloadPayload.total ??
+              downloadPayload.total_bytes ??
+              downloadPayload.totalBytes,
+          );
+          if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
+          const pct =
+            pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0;
+          perNodeMap.set(nodeId, {
+            nodeId,
+            nodeName,
+            status: pendingDownloaded > 0 ? "partial" : "pending",
+            percentage: pct,
+            progress: null,
+          });
+          continue;
+        }
 
+        // DownloadOngoing
         const progress = parseDownloadProgress(downloadPayload);
-        if (progress) {
-          // Sum all values across nodes - each node downloads independently
-          totalBytes += progress.totalBytes;
-          downloadedBytes += progress.downloadedBytes;
-          totalSpeed += progress.speed;
-          completedFiles += progress.completedFiles;
-          totalFiles += progress.totalFiles;
-          allFiles.push(...progress.files);
+        if (
+          !progress ||
+          (progress.downloadedBytes <= 0 && progress.totalBytes <= 0)
+        )
+          continue;
 
-          const nodeName =
-            data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
-          perNode.push({ nodeId, nodeName, progress });
-        }
+        perNodeMap.set(nodeId, {
+          nodeId,
+          nodeName,
+          status: "downloading",
+          percentage: progress.percentage,
+          progress,
+        });
+      }
+    }
+
+    // Aggregate from deduplicated per-node entries
+    const perNode = Array.from(perNodeMap.values());
+    let totalBytes = 0;
+    let downloadedBytes = 0;
+    let totalSpeed = 0;
+    let completedFiles = 0;
+    let totalFiles = 0;
+    let isDownloading = false;
+    const allFiles: DownloadProgress["files"] = [];
+
+    for (const node of perNode) {
+      if (node.status === "downloading" && node.progress) {
+        isDownloading = true;
+        totalBytes += node.progress.totalBytes;
+        downloadedBytes += node.progress.downloadedBytes;
+        totalSpeed += node.progress.speed;
+        completedFiles += node.progress.completedFiles;
+        totalFiles += node.progress.totalFiles;
+        allFiles.push(...node.progress.files);
       }
     }
 
     if (!isDownloading) {
-      return { isDownloading: false, progress: null, perNode: [] };
+      return {
+        isDownloading: false,
+        progress: null,
+        perNode,
+        failedError: null,
+      };
     }
 
-    // ETA = total remaining bytes / total speed across all nodes
     const remainingBytes = totalBytes - downloadedBytes;
     const etaMs = totalSpeed > 0 ? (remainingBytes / totalSpeed) * 1000 : 0;
 
@@ -1582,9 +1714,21 @@
         files: allFiles,
       },
       perNode,
+      failedError: null,
     };
   }
 
+  function getModelDownloadStatus(
+    modelId: string,
+    nodeIds?: string[],
+  ): {
+    isDownloading: boolean;
+    progress: DownloadProgress | null;
+    perNode: NodeDownloadStatus[];
+  } {
+    return collectDownloadStatus(modelId, nodeIds);
+  }
+
   // Helper to get download status for an instance
   function getInstanceDownloadStatus(
     instanceId: string,
@@ -1595,26 +1739,9 @@
     errorMessage: string | null;
     progress: DownloadProgress | null;
     statusText: string;
-    perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }>;
+    perNode: NodeDownloadStatus[];
   } {
-    if (!downloadsData || Object.keys(downloadsData).length === 0) {
-      // No download data yet — defer to runner status instead of assuming RUNNING
-      const statusInfo = deriveInstanceStatus(instanceWrapped);
-      return {
-        isDownloading: false,
-        isFailed: false,
-        errorMessage: null,
-        progress: null,
-        statusText: statusInfo.statusText,
-        perNode: [],
-      };
-    }
-
-    // Unwrap the instance
+    // Unwrap the instance to get shard assignments
     const [instanceTag, instance] = getTagged(instanceWrapped);
     if (!instance || typeof instance !== "object") {
       return {
@@ -1634,100 +1761,9 @@
         modelId?: string;
       };
     };
-    const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
-    const runnerToShard = inst.shardAssignments?.runnerToShard || {};
     const instanceModelId = inst.shardAssignments?.modelId;
 
-    // Build reverse mapping: runnerId -> nodeId
-    const runnerToNode: Record<string, string> = {};
-    for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) {
-      runnerToNode[runnerId] = nodeId;
-    }
-
-    let totalBytes = 0;
-    let downloadedBytes = 0;
-    let totalSpeed = 0;
-    let completedFiles = 0;
-    let totalFiles = 0;
-    let isDownloading = false;
-    const allFiles: DownloadProgress["files"] = [];
-    const perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }> = [];
-
-    // Check downloads for nodes that are part of this instance
-    for (const runnerId of Object.keys(runnerToShard)) {
-      const nodeId = runnerToNode[runnerId];
-      if (!nodeId) continue;
-
-      const nodeDownloads = downloadsData[nodeId];
-      if (!Array.isArray(nodeDownloads)) continue;
-
-      for (const downloadWrapped of nodeDownloads) {
-        if (!downloadWrapped || typeof downloadWrapped !== "object") continue;
-
-        const keys = Object.keys(downloadWrapped as Record<string, unknown>);
-        if (keys.length !== 1) continue;
-
-        const downloadKind = keys[0];
-        const downloadPayload = (downloadWrapped as Record<string, unknown>)[
-          downloadKind
-        ] as Record<string, unknown>;
-
-        // Handle DownloadFailed - return immediately with error info
-        if (downloadKind === "DownloadFailed") {
-          const downloadModelId = extractModelIdFromDownload(downloadPayload);
-          if (
-            instanceModelId &&
-            downloadModelId &&
-            downloadModelId === instanceModelId
-          ) {
-            return {
-              isDownloading: false,
-              isFailed: true,
-              errorMessage:
-                (downloadPayload.errorMessage as string) || "Download failed",
-              progress: null,
-              statusText: "FAILED",
-              perNode: [],
-            };
-          }
-        }
-
-        if (downloadKind !== "DownloadOngoing") continue;
-        if (!downloadPayload) continue;
-
-        // Check if this download is for this instance's model
-        const downloadModelId = extractModelIdFromDownload(downloadPayload);
-        if (
-          instanceModelId &&
-          downloadModelId &&
-          downloadModelId === instanceModelId
-        ) {
-          isDownloading = true;
-
-          const progress = parseDownloadProgress(downloadPayload);
-          if (progress) {
-            // Sum all values across nodes - each node downloads independently
-            totalBytes += progress.totalBytes;
-            downloadedBytes += progress.downloadedBytes;
-            totalSpeed += progress.speed;
-            completedFiles += progress.completedFiles;
-            totalFiles += progress.totalFiles;
-            allFiles.push(...progress.files);
-
-            const nodeName =
-              data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
-            perNode.push({ nodeId, nodeName, progress });
-          }
-        }
-      }
-    }
-
-    if (!isDownloading) {
-      // Check runner status for other states
+    if (!instanceModelId) {
       const statusInfo = deriveInstanceStatus(instanceWrapped);
       return {
         isDownloading: false,
@@ -1739,26 +1775,49 @@
       };
     }
 
-    // ETA = total remaining bytes / total speed across all nodes
-    const remainingBytes = totalBytes - downloadedBytes;
-    const etaMs = totalSpeed > 0 ? (remainingBytes / totalSpeed) * 1000 : 0;
+    // Get node IDs assigned to this instance
+    const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
+    const runnerToShard = inst.shardAssignments?.runnerToShard || {};
+    const runnerToNode: Record<string, string> = {};
+    for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) {
+      runnerToNode[runnerId] = nodeId;
+    }
+    const instanceNodeIds = Object.keys(runnerToShard)
+      .map((runnerId) => runnerToNode[runnerId])
+      .filter(Boolean);
+
+    const result = collectDownloadStatus(instanceModelId, instanceNodeIds);
+
+    if (result.failedError) {
+      return {
+        isDownloading: false,
+        isFailed: true,
+        errorMessage: result.failedError,
+        progress: null,
+        statusText: "FAILED",
+        perNode: [],
+      };
+    }
+
+    if (!result.isDownloading) {
+      const statusInfo = deriveInstanceStatus(instanceWrapped);
+      return {
+        isDownloading: false,
+        isFailed: statusInfo.statusText === "FAILED",
+        errorMessage: null,
+        progress: null,
+        statusText: statusInfo.statusText,
+        perNode: result.perNode,
+      };
+    }
 
     return {
       isDownloading: true,
       isFailed: false,
       errorMessage: null,
-      progress: {
-        totalBytes,
-        downloadedBytes,
-        speed: totalSpeed,
-        etaMs,
-        percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
-        completedFiles,
-        totalFiles,
-        files: allFiles,
-      },
+      progress: result.progress,
       statusText: "DOWNLOADING",
-      perNode,
+      perNode: result.perNode,
     };
   }
 
@@ -2786,7 +2845,7 @@
         // Running model is same or better tier — use it directly
         setSelectedChatModel(bestRunning.id);
         if (!chatStarted) createConversation();
-        sendMessage(content, files);
+        routeMessage(content, files);
         return;
       }
     }
@@ -2803,7 +2862,7 @@
     if (hasRunningInstance(autoModel.id)) {
       setSelectedChatModel(autoModel.id);
       if (!chatStarted) createConversation();
-      sendMessage(content, files);
+      routeMessage(content, files);
       return;
     }
 
@@ -2956,7 +3015,7 @@
       if (pendingAutoMessage) {
         const msg = pendingAutoMessage;
         pendingAutoMessage = null;
-        sendMessage(msg.content, msg.files);
+        routeMessage(msg.content, msg.files);
       }
       return;
     }
@@ -3035,7 +3094,7 @@
     // Model is selected and running — send directly
     if (model && hasRunningInstance(model)) {
       chatLaunchState = "ready";
-      sendMessage(content, files, null);
+      routeMessage(content, files);
       return;
     }
 
@@ -4516,7 +4575,7 @@
                 type="button"
                 onclick={() => {
                   completeOnboarding();
-                  sendMessage(chip);
+                  sendMessage(chip, undefined, thinkingEnabled());
                 }}
                 class="px-4 py-2 rounded-full border border-white/10 bg-white/5 text-sm text-white/60 hover:bg-white/10 hover:text-white/80 hover:border-white/20 transition-all duration-200 cursor-pointer"
               >
@@ -5255,10 +5314,10 @@
                             <div
                               class="mt-2 space-y-2 max-h-48 overflow-y-auto pr-1"
                             >
-                              {#each downloadInfo.perNode as nodeProg}
+                              {#each downloadInfo.perNode.filter((n) => n.status === "downloading" && n.progress) as nodeProg}
                                 {@const nodePercent = Math.min(
                                   100,
-                                  Math.max(0, nodeProg.progress.percentage),
+                                  Math.max(0, nodeProg.percentage),
                                 )}
                                 {@const isExpanded =
                                   instanceDownloadExpandedNodes.has(
@@ -5314,15 +5373,17 @@
                                     >
                                       <span
                                         >{formatBytes(
-                                          nodeProg.progress.downloadedBytes,
+                                          nodeProg.progress?.downloadedBytes ??
+                                            0,
                                         )} / {formatBytes(
-                                          nodeProg.progress.totalBytes,
+                                          nodeProg.progress?.totalBytes ?? 0,
                                         )}</span
                                       >
                                       <span
-                                        >{formatSpeed(nodeProg.progress.speed)} •
-                                        ETA {formatEta(
-                                          nodeProg.progress.etaMs,
+                                        >{formatSpeed(
+                                          nodeProg.progress?.speed ?? 0,
+                                        )} • ETA {formatEta(
+                                          nodeProg.progress?.etaMs ?? 0,
                                         )}</span
                                       >
                                     </div>
@@ -5330,14 +5391,14 @@
 
                                   {#if isExpanded}
                                     <div class="mt-2 space-y-1.5">
-                                      {#if nodeProg.progress.files.length === 0}
+                                      {#if nodeProg.progress?.files ?? [].length === 0}
                                         <div
                                           class="text-[11px] font-mono text-exo-light-gray/70"
                                         >
                                           No file details reported.
                                         </div>
                                       {:else}
-                                        {#each nodeProg.progress.files as f}
+                                        {#each nodeProg.progress?.files ?? [] as f}
                                           {@const filePercent = Math.min(
                                             100,
                                             Math.max(0, f.percentage ?? 0),
@@ -5813,12 +5874,15 @@
                 )}
                 {@const allPreviews = filteredPreviews()}
                 {#if selectedModel && allPreviews.length > 0}
-                  {@const downloadStatus = getModelDownloadStatus(
-                    selectedModel.id,
-                  )}
                   {@const tags = modelTags()[selectedModel.id] || []}
                   <div class="space-y-3">
                     {#each allPreviews as apiPreview, i}
+                      {@const downloadStatus = getModelDownloadStatus(
+                        selectedModel.id,
+                        apiPreview.memory_delta_by_node
+                          ? Object.keys(apiPreview.memory_delta_by_node)
+                          : undefined,
+                      )}
                       <div
                         role="group"
                         onmouseenter={() => {
@@ -6006,7 +6070,7 @@
                           onclick={() => {
                             chatLaunchState = "idle";
                             selectedChatCategory = null;
-                            sendMessage(prompt);
+                            sendMessage(prompt, undefined, thinkingEnabled());
                           }}
                           class="text-left px-3 py-2.5 text-xs text-exo-light-gray hover:text-white font-mono rounded-lg border border-exo-medium-gray/30 hover:border-exo-yellow/30 bg-exo-dark-gray/30 hover:bg-exo-dark-gray/60 transition-all duration-200 cursor-pointer"
                         >
@@ -6389,10 +6453,10 @@
                               <div
                                 class="mt-2 space-y-2 max-h-48 overflow-y-auto pr-1"
                               >
-                                {#each downloadInfo.perNode as nodeProg}
+                                {#each downloadInfo.perNode.filter((n) => n.status === "downloading" && n.progress) as nodeProg}
                                   {@const nodePercent = Math.min(
                                     100,
-                                    Math.max(0, nodeProg.progress.percentage),
+                                    Math.max(0, nodeProg.percentage),
                                   )}
                                   {@const isExpanded =
                                     instanceDownloadExpandedNodes.has(
@@ -6451,16 +6515,17 @@
                                       >
                                         <span
                                           >{formatBytes(
-                                            nodeProg.progress.downloadedBytes,
+                                            nodeProg.progress
+                                              ?.downloadedBytes ?? 0,
                                           )} / {formatBytes(
-                                            nodeProg.progress.totalBytes,
+                                            nodeProg.progress?.totalBytes ?? 0,
                                           )}</span
                                         >
                                         <span
                                           >{formatSpeed(
-                                            nodeProg.progress.speed,
+                                            nodeProg.progress?.speed ?? 0,
                                           )} • ETA {formatEta(
-                                            nodeProg.progress.etaMs,
+                                            nodeProg.progress?.etaMs ?? 0,
                                           )}</span
                                         >
                                       </div>
@@ -6468,14 +6533,14 @@
 
                                     {#if isExpanded}
                                       <div class="mt-2 space-y-1.5">
-                                        {#if nodeProg.progress.files.length === 0}
+                                        {#if nodeProg.progress?.files ?? [].length === 0}
                                           <div
                                             class="text-[11px] font-mono text-exo-light-gray/70"
                                           >
                                             No file details reported.
                                           </div>
                                         {:else}
-                                          {#each nodeProg.progress.files as f}
+                                          {#each nodeProg.progress?.files ?? [] as f}
                                             {@const filePercent = Math.min(
                                               100,
                                               Math.max(0, f.percentage ?? 0),

flake.nix

@@ -72,7 +72,7 @@
       ];
 
       perSystem =
-        { config, self', inputs', pkgs, lib, system, ... }:
+        { config, self', pkgs, lib, system, ... }:
         let
           # Use pinned nixpkgs for swift-format (swift is broken on x86_64-linux in newer nixpkgs)
           pkgsSwift = import inputs.nixpkgs-swift { inherit system; };
@@ -84,6 +84,17 @@
             config.allowUnfreePredicate = pkg: (pkg.pname or "") == "metal-toolchain";
             overlays = [
               (import ./nix/apple-sdk-overlay.nix)
+              (final: prev: {
+                macmon = prev.macmon.overrideAttrs (_: {
+                  version = "git";
+                  src = final.fetchFromGitHub {
+                    owner = "swiftraccoon";
+                    repo = "macmon";
+                    rev = "9154d234f763fbeffdcb4135d0bbbaf80609699b";
+                    hash = "sha256-CwhilKNbs5XL9/tF5DMwyPBlE/hpmjGNTuxQ36sM50M=";
+                  };
+                });
+              })
             ];
           };
           treefmt = {
@@ -117,12 +128,13 @@
               uvLock = builtins.fromTOML (builtins.readFile ./uv.lock);
               mlxPackage = builtins.head (builtins.filter (p: p.name == "mlx" && p.source ? git) uvLock.package);
               uvLockMlxVersion = mlxPackage.version;
+              uvLockMlxRev = builtins.elemAt (builtins.split "#" mlxPackage.source.git) 2;
             in
             {
               metal-toolchain = pkgs.callPackage ./nix/metal-toolchain.nix { };
               mlx = pkgs.callPackage ./nix/mlx.nix {
                 inherit (self'.packages) metal-toolchain;
-                inherit uvLockMlxVersion;
+                inherit uvLockMlxVersion uvLockMlxRev;
               };
               default = self'.packages.exo;
             }

justfile

@@ -1,5 +1,8 @@
 export NIX_CONFIG := "extra-experimental-features = nix-command flakes"
 
+default: lint fmt
+all: lint fmt check
+
 fmt:
     treefmt || nix fmt
 
@@ -31,6 +34,10 @@ build-dashboard:
 package:
     uv run pyinstaller packaging/pyinstaller/exo.spec
 
+build-app: package
+    xcodebuild build -project app/EXO/EXO.xcodeproj -scheme EXO -configuration Debug -derivedDataPath app/EXO/build
+    @echo "\nBuild complete. Run with:\n  open {{justfile_directory()}}/app/EXO/build/Build/Products/Debug/EXO.app"
+
 clean:
     rm -rf **/__pycache__
     rm -rf target/

nix/mlx.nix

@@ -11,6 +11,7 @@
 , fmt
 , python313Packages
 , uvLockMlxVersion
+, uvLockMlxRev
 }:
 
 assert stdenv.isDarwin;
@@ -41,15 +42,13 @@ let
 
   mlx = stdenv.mkDerivation rec {
     pname = "mlx";
-    version = let v = "0.30.7.dev20260225+257d5692"; in
-      assert v == uvLockMlxVersion || throw "MLX version mismatch: nix/mlx.nix has ${v} but uv.lock has ${uvLockMlxVersion}. Update both the version and hash in nix/mlx.nix.";
-      v;
+    version = uvLockMlxVersion;
     pyproject = true;
 
     src = fetchFromGitHub {
       owner = "rltakashige";
       repo = "mlx-jaccl-fix-small-recv";
-      rev = "257d5692fc7af6bba3b8afaeb63c549b7d1e43d5";
+      rev = uvLockMlxRev;
       hash = "sha256-GosFIWxIB48Egb1MqJrR3xhsUsQeWdRk5rV93USY6wQ=";
     };
 

packaging/pyinstaller/exo.spec

@@ -71,7 +71,9 @@ MACMON_PATH = shutil.which("macmon")
 if MACMON_PATH is None:
     raise SystemExit(
         "macmon binary not found in PATH. "
-        "Install it via: brew install macmon"
+        "Install the pinned fork used by exo via: "
+        "cargo install --git https://github.com/swiftraccoon/macmon "
+        "--rev 9154d234f763fbeffdcb4135d0bbbaf80609699b macmon --force"
     )
 
 BINARIES: list[tuple[str, str]] = [
@@ -120,4 +122,3 @@ coll = COLLECT(
     upx_exclude=[],
     name="exo",
 )
-

pyproject.toml

@@ -25,8 +25,7 @@ dependencies = [
   "openai-harmony>=0.0.8",
   "httpx>=0.28.1",
   "tomlkit>=0.14.0",
-  "pillow>=11.0,<12.0",                        # compatibility with mflux
-  "mflux==0.15.5",
+  "mflux==0.17.2",
   "python-multipart>=0.0.21",
   "msgspec>=0.19.0",
   "zstandard>=0.23.0",
@@ -62,7 +61,7 @@ members = ["rust/exo_pyo3_bindings", "bench"]
 [tool.uv.sources]
 exo_pyo3_bindings = { workspace = true }
 mlx = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git", branch = "address-rdma-gpu-locks", marker = "sys_platform == 'darwin'" }
-mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/eval-left-padding-in-batched-rotation" }
+mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/fix-deepseek-v32-indexer" }
 # Uncomment to use local mlx/mlx-lm development versions:
 # mlx = { path = "/Users/Shared/mlx", editable=true }
 # mlx-lm = { path = "/Users/Shared/mlx-lm", editable=true }

resources/inference_model_cards/mlx-community--DeepSeek-V3.1-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/DeepSeek-V3.1-4bit"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 128
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "deepseek"

resources/inference_model_cards/mlx-community--DeepSeek-V3.1-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/DeepSeek-V3.1-8bit"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 128
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "deepseek"

resources/inference_model_cards/mlx-community--DeepSeek-V3.2-4bit.toml

@@ -0,0 +1,13 @@
+model_id = "mlx-community/DeepSeek-V3.2-4bit"
+n_layers = 61
+hidden_size = 7168
+num_key_value_heads = 128
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "deepseek"
+quantization = "4bit"
+base_model = "DeepSeek V3.2"
+capabilities = ["text", "thinking", "thinking_toggle"]
+
+[storage_size]
+in_bytes = 378086226621

resources/inference_model_cards/mlx-community--DeepSeek-V3.2-8bit.toml

@@ -0,0 +1,13 @@
+model_id = "mlx-community/DeepSeek-V3.2-8bit"
+n_layers = 61
+hidden_size = 7168
+num_key_value_heads = 128
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "deepseek"
+quantization = "8bit"
+base_model = "DeepSeek V3.2"
+capabilities = ["text", "thinking", "thinking_toggle"]
+
+[storage_size]
+in_bytes = 755957120916

resources/inference_model_cards/mlx-community--GLM-4.5-Air-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.5-Air-8bit"
 n_layers = 46
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = false
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-4.5-Air-bf16.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.5-Air-bf16"
 n_layers = 46
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-4.7-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-4bit"
 n_layers = 91
 hidden_size = 5120
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-4.7-6bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-6bit"
 n_layers = 91
 hidden_size = 5120
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-4.7-8bit-gs32.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-8bit-gs32"
 n_layers = 91
 hidden_size = 5120
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-4.7-Flash-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-Flash-4bit"
 n_layers = 47
 hidden_size = 2048
+num_key_value_heads = 20
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-4.7-Flash-5bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-Flash-5bit"
 n_layers = 47
 hidden_size = 2048
+num_key_value_heads = 20
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-4.7-Flash-6bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-Flash-6bit"
 n_layers = 47
 hidden_size = 2048
+num_key_value_heads = 20
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-4.7-Flash-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-Flash-8bit"
 n_layers = 47
 hidden_size = 2048
+num_key_value_heads = 20
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-5-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-5-8bit-MXFP8"
 n_layers = 78
 hidden_size = 6144
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-5-MXFP4-Q8.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-5-MXFP4-Q8"
 n_layers = 78
 hidden_size = 6144
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--GLM-5-bf16.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-5"
 n_layers = 78
 hidden_size = 6144
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"

resources/inference_model_cards/mlx-community--Kimi-K2-Instruct-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Kimi-K2-Instruct-4bit"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "kimi"

resources/inference_model_cards/mlx-community--Kimi-K2-Thinking.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Kimi-K2-Thinking"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "kimi"

resources/inference_model_cards/mlx-community--Kimi-K2.5.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Kimi-K2.5"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "kimi"

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-70B-Instruct-HF-4bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-70B-Instruct-HF-4bit"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "4bit"
+base_model = "NVIDIA Llama-3.1-Nemotron-70B-Instruct"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 39688355840

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-70B-Instruct-HF-8bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-70B-Instruct-HF-8bit"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "8bit"
+base_model = "NVIDIA Llama-3.1-Nemotron-70B-Instruct"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 74964549632

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-70B-Instruct-HF-bf16.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-70B-Instruct-HF-bf16"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "bf16"
+base_model = "NVIDIA Llama-3.1-Nemotron-70B-Instruct"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 141107412992

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-Nano-4B-v1.1-4bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-Nano-4B-v1.1-4bit"
+n_layers = 32
+hidden_size = 3072
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "4bit"
+base_model = "NVIDIA Llama-3.1-Nemotron-Nano-4B-v1.1"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 2538706944

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-Nano-4B-v1.1-8bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-Nano-4B-v1.1-8bit"
+n_layers = 32
+hidden_size = 3072
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "8bit"
+base_model = "NVIDIA Llama-3.1-Nemotron-Nano-4B-v1.1"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 4794980352

resources/inference_model_cards/mlx-community--Llama-3.1-Nemotron-Nano-4B-v1.1-bf16.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-Nano-4B-v1.1-bf16"
+n_layers = 32
+hidden_size = 3072
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "bf16"
+base_model = "NVIDIA Llama-3.1-Nemotron-Nano-4B-v1.1"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 9025492992

resources/inference_model_cards/mlx-community--Llama-3.2-1B-Instruct-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.2-1B-Instruct-4bit"
 n_layers = 16
 hidden_size = 2048
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--Llama-3.2-3B-Instruct-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.2-3B-Instruct-4bit"
 n_layers = 28
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--Llama-3.2-3B-Instruct-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.2-3B-Instruct-8bit"
 n_layers = 28
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--Llama-3.3-70B-Instruct-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.3-70B-Instruct-4bit"
 n_layers = 80
 hidden_size = 8192
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--Llama-3.3-70B-Instruct-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.3-70B-Instruct-8bit"
 n_layers = 80
 hidden_size = 8192
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--Meta-Llama-3.1-70B-Instruct-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"
 n_layers = 80
 hidden_size = 8192
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--Meta-Llama-3.1-8B-Instruct-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"
 n_layers = 32
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--Meta-Llama-3.1-8B-Instruct-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"
 n_layers = 32
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--Meta-Llama-3.1-8B-Instruct-bf16.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"
 n_layers = 32
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"

resources/inference_model_cards/mlx-community--MiniMax-M2.1-3bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.1-3bit"
 n_layers = 61
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"

resources/inference_model_cards/mlx-community--MiniMax-M2.1-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.1-8bit"
 n_layers = 61
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"

resources/inference_model_cards/mlx-community--MiniMax-M2.5-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.5-4bit"
 n_layers = 62
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"

resources/inference_model_cards/mlx-community--MiniMax-M2.5-6bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.5-6bit"
 n_layers = 62
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"

resources/inference_model_cards/mlx-community--MiniMax-M2.5-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.5-8bit"
 n_layers = 62
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-4Bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-4Bit"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "4bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 17775342336

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-5Bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-5Bit"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "5bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 21721476864

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-6Bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-6Bit"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "6bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 25667611392

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-8Bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-8Bit"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "8bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 33559880448

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-BF16.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-BF16"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "bf16"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 63155889408

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-MXFP4.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-MXFP4"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "4bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 16788808704

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "4bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 19323906944

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-Nano-9B-v2-4bits.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-Nano-9B-v2-4bits"
+n_layers = 56
+hidden_size = 4480
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "4bit"
+base_model = "NVIDIA Nemotron-Nano-9B-v2"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 5002791936

resources/inference_model_cards/mlx-community--NVIDIA-Nemotron-Nano-9B-v2-6bit.toml

@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-Nano-9B-v2-6bit"
+n_layers = 56
+hidden_size = 4480
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "6bit"
+base_model = "NVIDIA Nemotron-Nano-9B-v2"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 7224298496

resources/inference_model_cards/mlx-community--Qwen3-0.6B-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-0.6B-4bit"
 n_layers = 28
 hidden_size = 1024
+num_key_value_heads = 8
 supports_tensor = false
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-0.6B-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-0.6B-8bit"
 n_layers = 28
 hidden_size = 1024
+num_key_value_heads = 8
 supports_tensor = false
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-235B-A22B-Instruct-2507-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"
 n_layers = 94
 hidden_size = 4096
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-235B-A22B-Instruct-2507-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"
 n_layers = 94
 hidden_size = 4096
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-30B-A3B-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-30B-A3B-4bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-30B-A3B-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-30B-A3B-8bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Coder-480B-A35B-Instruct-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"
 n_layers = 62
 hidden_size = 6144
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Coder-480B-A35B-Instruct-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"
 n_layers = 62
 hidden_size = 6144
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-4bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-5bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-5bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-6bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-6bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-8bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Coder-Next-bf16.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-bf16"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Next-80B-A3B-Instruct-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Next-80B-A3B-Instruct-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Next-80B-A3B-Thinking-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3-Next-80B-A3B-Thinking-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3.5-122B-A10B-4bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-122B-A10B-4bit"
 n_layers = 48
 hidden_size = 3072
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3.5-122B-A10B-6bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-122B-A10B-6bit"
 n_layers = 48
 hidden_size = 3072
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"

resources/inference_model_cards/mlx-community--Qwen3.5-122B-A10B-8bit.toml

@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-122B-A10B-8bit"
 n_layers = 48
 hidden_size = 3072
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"