persist node ids in .cache

brings back EXO_CACHE_HOME as always ~/.cache/exo/, and store the node id in there. no random copies now!
Update mflux to 0.16.9 (#1751 )
2026-03-18 11:24:04 +00:00 · 2026-03-17 16:58:23 +00:00 · 2026-03-17 13:47:24 +00:00 · 2026-03-17 13:36:44 +00:00 · 2026-03-17 11:22:01 +00:00 · 2026-03-13 14:55:37 +00:00
117 changed files with 1354 additions and 351 deletions
@@ -32,6 +32,7 @@ class Conv1d(Module):
    """

    weight: mx.array
+    bias: mx.array | None
    groups: int
    def __init__(
        self,
@@ -40,6 +40,10 @@ class Linear(Module):
        bias (bool, optional): If set to ``False`` then the layer will
          not use a bias. Default is ``True``.
    """
+
+    weight: mx.array
+    bias: mx.array | None
+
    def __init__(self, input_dims: int, output_dims: int, bias: bool = ...) -> None: ...
    def __call__(self, x: mx.array) -> mx.array: ...
    def to_quantized(
@@ -88,6 +88,9 @@ class RMSNorm(Module):
        dims (int): The feature dimension of the input to normalize over
        eps (float): A small additive constant for numerical stability
    """
+
+    weight: mx.array
+
    def __init__(self, dims: int, eps: float = ...) -> None: ...
    def __call__(self, x) -> mx.array: ...

@@ -0,0 +1,154 @@
+from dataclasses import dataclass
+from typing import Any, List, Optional, Tuple
+
+import mlx.core as mx
+import mlx.nn as nn
+
+from .cache import ArraysCache, KVCache
+from .switch_layers import SwitchMLP
+
+@dataclass
+class ModelArgs:
+    model_type: str
+    vocab_size: int
+    hidden_size: int
+    intermediate_size: int
+    num_hidden_layers: int
+    max_position_embeddings: int
+    num_attention_heads: int
+    num_key_value_heads: int
+    attention_bias: bool
+    mamba_num_heads: int
+    mamba_head_dim: int
+    mamba_proj_bias: bool
+    ssm_state_size: int
+    conv_kernel: int
+    n_groups: int
+    mlp_bias: bool
+    layer_norm_epsilon: float
+    use_bias: bool
+    use_conv_bias: bool
+    hybrid_override_pattern: List[str]
+    head_dim: Optional[int]
+    moe_intermediate_size: Optional[int]
+    moe_shared_expert_intermediate_size: Optional[int]
+    n_group: Optional[int]
+    n_routed_experts: Optional[int]
+    n_shared_experts: Optional[int]
+    topk_group: Optional[int]
+    num_experts_per_tok: Optional[int]
+    norm_topk_prob: Optional[bool]
+    routed_scaling_factor: Optional[float]
+    time_step_limit: Optional[Tuple[float, float]]
+    time_step_min: Optional[float]
+    time_step_max: Optional[float]
+
+    @classmethod
+    def from_dict(cls, params: dict[str, Any]) -> ModelArgs: ...
+    def __post_init__(self) -> None: ...
+
+class NemotronHMamba2Mixer(nn.Module):
+    num_heads: int
+    hidden_size: int
+    ssm_state_size: int
+    conv_kernel_size: int
+    intermediate_size: int
+    n_groups: int
+    head_dim: int
+    conv_dim: int
+    conv1d: nn.Conv1d
+    in_proj: nn.Linear
+    dt_bias: mx.array
+    A_log: mx.array
+    D: mx.array
+    norm: nn.RMSNorm
+    heads_per_group: int
+    out_proj: nn.Linear
+
+    def __init__(self, args: ModelArgs) -> None: ...
+    def __call__(
+        self,
+        hidden_states: mx.array,
+        mask: Optional[mx.array],
+        cache: Optional[ArraysCache] = None,
+    ) -> mx.array: ...
+
+class NemotronHAttention(nn.Module):
+    hidden_size: int
+    num_heads: int
+    head_dim: int
+    num_key_value_heads: int
+    scale: float
+    q_proj: nn.Linear
+    k_proj: nn.Linear
+    v_proj: nn.Linear
+    o_proj: nn.Linear
+
+    def __init__(self, args: ModelArgs) -> None: ...
+    def __call__(
+        self,
+        x: mx.array,
+        mask: Optional[mx.array] = None,
+        cache: Optional[KVCache] = None,
+    ) -> mx.array: ...
+
+class NemotronHMLP(nn.Module):
+    up_proj: nn.Linear
+    down_proj: nn.Linear
+
+    def __init__(
+        self, args: ModelArgs, intermediate_size: Optional[int] = None
+    ) -> None: ...
+    def __call__(self, x: mx.array) -> mx.array: ...
+
+class NemotronHMoE(nn.Module):
+    num_experts_per_tok: int
+    switch_mlp: SwitchMLP
+    shared_experts: NemotronHMLP
+
+    def __init__(self, config: ModelArgs) -> None: ...
+    def __call__(self, x: mx.array) -> mx.array: ...
+
+class NemotronHBlock(nn.Module):
+    block_type: str
+    norm: nn.RMSNorm
+    mixer: NemotronHMamba2Mixer | NemotronHAttention | NemotronHMLP | NemotronHMoE
+
+    def __init__(self, args: ModelArgs, block_type: str) -> None: ...
+    def __call__(
+        self,
+        x: mx.array,
+        mask: Optional[mx.array] = None,
+        cache: Optional[Any] = None,
+    ) -> mx.array: ...
+
+class NemotronHModel(nn.Module):
+    embeddings: nn.Embedding
+    layers: list[NemotronHBlock]
+    norm_f: nn.RMSNorm
+    fa_idx: int
+    ssm_idx: int
+
+    def __init__(self, args: ModelArgs) -> None: ...
+    def __call__(
+        self,
+        inputs: mx.array,
+        cache: Optional[Any] = None,
+    ) -> mx.array: ...
+
+class Model(nn.Module):
+    args: ModelArgs
+    backbone: NemotronHModel
+    lm_head: nn.Linear
+    model_type: str
+
+    def __init__(self, args: ModelArgs) -> None: ...
+    def __call__(
+        self,
+        inputs: mx.array,
+        cache: Optional[Any] = None,
+    ) -> mx.array: ...
+    @property
+    def layers(self) -> list[NemotronHBlock]: ...
+    def make_cache(self) -> list[ArraysCache | KVCache]: ...
+    def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
@@ -5,6 +5,7 @@ from typing import Any, Optional
 import mlx.core as mx
 import mlx.nn as nn

+from .cache import ArraysCache, KVCache
 from .switch_layers import SwitchGLU

 class Qwen3NextRMSNormGated(nn.Module):
@@ -99,6 +100,8 @@ class Qwen3NextModel(nn.Module):
    embed_tokens: nn.Embedding
    layers: list[Qwen3NextDecoderLayer]
    norm: nn.RMSNorm
+    ssm_idx: int
+    fa_idx: int

    def __init__(self, args: Any) -> None: ...
    def __call__(
@@ -121,3 +124,4 @@ class Model(nn.Module):
    def sanitize(self, weights: dict[str, Any]) -> dict[str, Any]: ...
    @property
    def layers(self) -> list[Qwen3NextDecoderLayer]: ...
+    def make_cache(self) -> list[ArraysCache | KVCache]: ...
@@ -73,6 +73,9 @@ class SwitchGLU(nn.Module):
    def __call__(self, x, indices) -> mx.array: ...

 class SwitchMLP(nn.Module):
+    fc1: SwitchLinear
+    fc2: SwitchLinear
+
    def __init__(
        self,
        input_dims: int,
@@ -39,11 +39,11 @@ class StreamingDetokenizer:
    """

    __slots__ = ...
-    def reset(self): ...
-    def add_token(self, token): ...
-    def finalize(self): ...
+    def reset(self) -> None: ...
+    def add_token(self, token: int) -> None: ...
+    def finalize(self) -> None: ...
    @property
-    def last_segment(self):
+    def last_segment(self) -> str:
        """Return the last segment of readable text since last time this property was accessed."""

 class NaiveStreamingDetokenizer(StreamingDetokenizer):
@@ -496,20 +496,28 @@ def main() -> int:
                                all_rows.append(row)

                            if batch_results:
-                                agg_gen_tps = sum(
+                                valid_gen_tps = [
                                    x["stats"]["generation_tps"]
                                    for x, _ in batch_results
                                    if x["stats"]["generation_tps"] > 0
+                                ]
+                                agg_gen_tps = (
+                                    mean(valid_gen_tps) if valid_gen_tps else 0.0
                                )
+                                gen_tps = agg_gen_tps / concurrency
                                logger.info(
                                    f"[concurrent {concurrency}x]  "
                                    f"agg_gen_tps={agg_gen_tps:.2f}  "
+                                    f"gen_tps={gen_tps:.2f}  "
                                    f"errors={batch_errors}"
                                )

                    if runs:
                        prompt_tps = mean(x["stats"]["prompt_tps"] for x in runs)
-                        gen_tps = mean(x["stats"]["generation_tps"] for x in runs)
+                        gen_tps = mean(
+                            x["stats"]["generation_tps"] / x["concurrency"]
+                            for x in runs
+                        )
                        ptok = mean(x["stats"]["prompt_tokens"] for x in runs)
                        gtok = mean(x["stats"]["generation_tokens"] for x in runs)
                        peak = mean(
@@ -85,9 +85,6 @@ _MC_PATTERNS: list[re.Pattern[str]] = [
 # Code extraction: last ```python ... ``` block (AA regex)
 _CODE_BLOCK_RE = re.compile(r"```(?:python|Python)?\s*\n(.*?)```", re.DOTALL)

-# LCB-compatible extraction mode (--lcb-compat): use line-based extraction
-# matching official LiveCodeBench extract_code() behavior.
-

 # ---------------------------------------------------------------------------
 # Model config loading
@@ -151,22 +148,12 @@ def extract_boxed_answer(text: str) -> str | None:
    return matches[-1].strip() if matches else None


-def extract_code_block(text: str, preserve_indent: bool = False, lcb_compat: bool = False) -> str | None:
+def extract_code_block(text: str, preserve_indent: bool = False) -> str | None:
    """Extract the last Python code block from markdown response.

    If preserve_indent is True, only strip trailing whitespace (keeps leading
    indentation intact — needed for HumanEval function-body completions).
-
-    If lcb_compat is True, use the official LiveCodeBench extract_code() logic:
-    line-based search for ```, extract between last two backtick lines.
    """
-    if lcb_compat:
-        lines = text.split("\n")
-        backtick_lines = [i for i, line in enumerate(lines) if "```" in line]
-        if len(backtick_lines) < 2:
-            return ""
-        return "\n".join(lines[backtick_lines[-2] + 1 : backtick_lines[-1]])
-
    matches = _CODE_BLOCK_RE.findall(text)
    if matches:
        raw = matches[-1]
@@ -397,7 +384,7 @@ _LCB_WITH_STARTER = (
    "### Format: You will use the following starter code to write the "
    "solution to the problem and enclose your code within delimiters.\n"
    "```python\n{starter_code}\n```\n\n"
-    "### Answer: (use the provided format with backticks)\n\n"
+    "### Answer: (use the provided format with backticks)\n"
 )

 _LCB_WITHOUT_STARTER = (
@@ -408,7 +395,7 @@ _LCB_WITHOUT_STARTER = (
    "python program runs, it reads the inputs, runs the algorithm and "
    "writes output to STDOUT.\n"
    "```python\n# YOUR CODE HERE\n```\n\n"
-    "### Answer: (use the provided format with backticks)\n\n"
+    "### Answer: (use the provided format with backticks)\n"
 )


@@ -542,11 +529,6 @@ async def _call_api(
    system_message: str | None = None,
    reasoning_effort: str | None = None,
    top_p: float | None = None,
-    enable_thinking: bool | None = None,
-    top_k: int | None = None,
-    min_p: float | None = None,
-    repetition_penalty: float | None = None,
-    repetition_context_size: int | None = None,
 ) -> ApiResult:
    messages = []
    if system_message:
@@ -563,15 +545,6 @@ async def _call_api(
        body["reasoning_effort"] = reasoning_effort
    if top_p is not None:
        body["top_p"] = top_p
-    if enable_thinking is not None:
-        body["enable_thinking"] = enable_thinking
-    if top_k is not None:
-        body["top_k"] = top_k
-    if min_p is not None:
-        body["min_p"] = min_p
-    if repetition_penalty is not None:
-        body["repetition_penalty"] = repetition_penalty
-        body["repetition_context_size"] = repetition_context_size or 64

    resp = await client.post(
        f"{base_url}/v1/chat/completions",
@@ -604,11 +577,6 @@ async def call_with_retries(
    system_message: str | None = None,
    reasoning_effort: str | None = None,
    top_p: float | None = None,
-    enable_thinking: bool | None = None,
-    top_k: int | None = None,
-    min_p: float | None = None,
-    repetition_penalty: float | None = None,
-    repetition_context_size: int | None = None,
 ) -> ApiResult | None:
    for attempt in range(MAX_RETRIES):
        try:
@@ -623,11 +591,6 @@ async def call_with_retries(
                system_message,
                reasoning_effort,
                top_p,
-                enable_thinking,
-                top_k,
-                min_p,
-                repetition_penalty,
-                repetition_context_size,
            )
        except Exception as e:
            if attempt < MAX_RETRIES - 1:
@@ -658,14 +621,6 @@ async def evaluate_benchmark(
    reasoning_effort: str | None = None,
    top_p: float | None = None,
    difficulty: str | None = None,
-    start_index: int | None = None,
-    end_index: int | None = None,
-    lcb_compat: bool = False,
-    enable_thinking: bool | None = None,
-    top_k: int | None = None,
-    min_p: float | None = None,
-    repetition_penalty: float | None = None,
-    repetition_context_size: int | None = None,
 ) -> list[QuestionResult]:
    """Run a benchmark. Returns per-question results."""
    import datasets
@@ -680,8 +635,6 @@ async def evaluate_benchmark(
                data_files="hf://datasets/livecodebench/code_generation_lite/*.jsonl",
                split="train",
            )
-            # Sort by question_id to match official LCB runner ordering
-            ds = ds.sort("question_id")
        else:
            ds = datasets.load_dataset(
                config.dataset_name,
@@ -698,12 +651,6 @@ async def evaluate_benchmark(
        ds = ds.filter(lambda x: x["difficulty"] == difficulty)
        logger.info(f"Filtered to {len(ds)} {difficulty} problems")

-    if start_index is not None or end_index is not None:
-        si = start_index or 0
-        ei = end_index or len(ds)
-        ds = ds.select(range(si, min(ei, len(ds))))
-        logger.info(f"Sliced to [{si}:{ei}] → {len(ds)} problems")
-
    total = len(ds)
    if limit and limit < total:
        ds = ds.select(range(limit))
@@ -761,11 +708,6 @@ async def evaluate_benchmark(
                system_message=system_msg,
                reasoning_effort=reasoning_effort,
                top_p=top_p,
-                enable_thinking=enable_thinking,
-                top_k=top_k,
-                min_p=min_p,
-                repetition_penalty=repetition_penalty,
-                repetition_context_size=repetition_context_size,
            )
            elapsed = time.monotonic() - t0

@@ -817,9 +759,8 @@ async def evaluate_benchmark(
            elif config.kind == "code":
                # HumanEval needs preserved indentation (function body completion)
                keep_indent = benchmark_name == "humaneval"
-                code = extract_code_block(response, preserve_indent=keep_indent,
-                                          lcb_compat=(lcb_compat and benchmark_name == "livecodebench"))
-                if not code:
+                code = extract_code_block(response, preserve_indent=keep_indent)
+                if code is None:
                    result = QuestionResult(
                        question_id=idx,
                        prompt=prompt,
@@ -1152,18 +1093,6 @@ def main() -> int:
        default=None,
        help="Max questions per benchmark (for fast iteration).",
    )
-    ap.add_argument(
-        "--start-index",
-        type=int,
-        default=None,
-        help="Start index for problem range (inclusive). Applied after difficulty filter.",
-    )
-    ap.add_argument(
-        "--end-index",
-        type=int,
-        default=None,
-        help="End index for problem range (exclusive). Applied after difficulty filter.",
-    )

    reasoning_group = ap.add_mutually_exclusive_group()
    reasoning_group.add_argument(
@@ -1226,26 +1155,6 @@ def main() -> int:
        action="store_true",
        help="Skip exo instance management (assumes model is already running).",
    )
-    ap.add_argument(
-        "--lcb-compat",
-        action="store_true",
-        help="Use LiveCodeBench-compatible code extraction and prompt format.",
-    )
-    ap.add_argument(
-        "--enable-thinking",
-        action="store_true",
-        default=False,
-        help="Send enable_thinking=true in API request (for Qwen/DeepSeek thinking mode).",
-    )
-    ap.add_argument("--top-k", type=int, default=None, help="Override top_k sampling.")
-    ap.add_argument("--min-p", type=float, default=None, help="Override min_p sampling.")
-    ap.add_argument(
-        "--repetition-penalty", type=float, default=None, help="Override repetition_penalty."
-    )
-    ap.add_argument(
-        "--repetition-context-size", type=int, default=None,
-        help="Context window for repetition penalty (default: 64 when repetition_penalty is set).",
-    )

    args, _ = ap.parse_known_args()

@@ -1377,29 +1286,12 @@ def main() -> int:
        reasoning_effort = str(cfg["reasoning_effort"])
    else:
        reasoning_effort = "high" if is_reasoning else None
-
-    enable_thinking = True if args.enable_thinking else None
-
-    top_k: int | None = args.top_k
-    min_p: float | None = args.min_p
-    repetition_penalty: float | None = args.repetition_penalty
-    repetition_context_size: int | None = args.repetition_context_size
-
    base_url = f"http://{args.host}:{args.port}"

    logger.info(f"Model: {full_model_id}")
-    extra_params = ""
-    if top_p is not None:
-        extra_params += f"top_p={top_p}, "
-    if top_k is not None:
-        extra_params += f"top_k={top_k}, "
-    if min_p is not None:
-        extra_params += f"min_p={min_p}, "
-    if repetition_penalty is not None:
-        extra_params += f"repetition_penalty={repetition_penalty}, "
    logger.info(
        f"Settings: temperature={temperature}, max_tokens={max_tokens}, "
-        + extra_params
+        + (f"top_p={top_p}, " if top_p is not None else "")
        + f"reasoning={'yes' if is_reasoning else 'no'}"
        + (f", reasoning_effort={reasoning_effort}" if reasoning_effort else "")
    )
@@ -1425,14 +1317,6 @@ def main() -> int:
                            reasoning_effort=reasoning_effort,
                            top_p=top_p,
                            difficulty=args.difficulty,
-                            start_index=args.start_index,
-                            end_index=args.end_index,
-                            lcb_compat=args.lcb_compat,
-                            enable_thinking=enable_thinking,
-                            top_k=top_k,
-                            min_p=min_p,
-                            repetition_penalty=repetition_penalty,
-                            repetition_context_size=repetition_context_size,
                        )
                    )
                    if results:
@@ -1463,14 +1347,6 @@ def main() -> int:
                        reasoning_effort=reasoning_effort,
                        top_p=top_p,
                        difficulty=args.difficulty,
-                        start_index=args.start_index,
-                        end_index=args.end_index,
-                        lcb_compat=args.lcb_compat,
-                        enable_thinking=enable_thinking,
-                        top_k=top_k,
-                        min_p=min_p,
-                        repetition_penalty=repetition_penalty,
-                        repetition_context_size=repetition_context_size,
                    )
                )
                if results:
@@ -1,9 +1,6 @@
 <script lang="ts">
  import {
    isLoading,
-    sendMessage,
-    generateImage,
-    editImage,
    editingImage,
    clearEditingImage,
    selectedChatModel,
@@ -28,7 +25,7 @@
    modelTasks?: Record<string, string[]>;
    modelCapabilities?: Record<string, string[]>;
    onSend?: () => void;
-    onAutoSend?: (
+    onAutoSend: (
      content: string,
      files?: {
        id: string;
@@ -216,49 +213,10 @@
    uploadedFiles = [];
    resetTextareaHeight();

-    // When onAutoSend is provided, the parent controls all send logic
-    // (including launching non-running models before sending)
-    if (onAutoSend) {
-      onAutoSend(content, files);
-      onSend?.();
-      setTimeout(() => textareaRef?.focus(), 10);
-      return;
-    }
-
-    // Use image editing if in edit mode
-    if (isEditMode && currentEditingImage && content) {
-      editImage(content, currentEditingImage.imageDataUrl);
-    }
-    // If user attached an image with an ImageToImage model, use edit endpoint
-    else if (
-      currentModel &&
-      modelSupportsImageEditing(currentModel) &&
-      files.length > 0 &&
-      content
-    ) {
-      // Use the first attached image for editing
-      const imageFile = files[0];
-      if (imageFile.preview) {
-        editImage(content, imageFile.preview);
-      }
-    } else if (
-      currentModel &&
-      modelSupportsTextToImage(currentModel) &&
-      content
-    ) {
-      // Use image generation for text-to-image models
-      generateImage(content);
-    } else {
-      sendMessage(
-        content,
-        files,
-        modelSupportsThinking() ? thinkingEnabled : null,
-      );
-    }
-
+    // Parent controls all send logic (including image routing,
+    // launching non-running models before sending, etc.)
+    onAutoSend(content, files);
    onSend?.();
-
-    // Refocus the textarea after sending
    setTimeout(() => textareaRef?.focus(), 10);
  }

@@ -82,6 +82,12 @@
      d="M12.025 1.13c-5.77 0-10.449 4.647-10.449 10.378 0 1.112.178 2.181.503 3.185.064-.222.203-.444.416-.577a.96.96 0 0 1 .524-.15c.293 0 .584.124.84.284.278.173.48.408.71.694.226.282.458.611.684.951v-.014c.017-.324.106-.622.264-.874s.403-.487.762-.543c.3-.047.596.06.787.203s.31.313.4.467c.15.257.212.468.233.542.01.026.653 1.552 1.657 2.54.616.605 1.01 1.223 1.082 1.912.055.537-.096 1.059-.38 1.572.637.121 1.294.187 1.967.187.657 0 1.298-.063 1.921-.178-.287-.517-.44-1.041-.384-1.581.07-.69.465-1.307 1.081-1.913 1.004-.987 1.647-2.513 1.657-2.539.021-.074.083-.285.233-.542.09-.154.208-.323.4-.467a1.08 1.08 0 0 1 .787-.203c.359.056.604.29.762.543s.247.55.265.874v.015c.225-.34.457-.67.683-.952.23-.286.432-.52.71-.694.257-.16.547-.284.84-.285a.97.97 0 0 1 .524.151c.228.143.373.388.43.625l.006.04a10.3 10.3 0 0 0 .534-3.273c0-5.731-4.678-10.378-10.449-10.378M8.327 6.583a1.5 1.5 0 0 1 .713.174 1.487 1.487 0 0 1 .617 2.013c-.183.343-.762-.214-1.102-.094-.38.134-.532.914-.917.71a1.487 1.487 0 0 1 .69-2.803m7.486 0a1.487 1.487 0 0 1 .689 2.803c-.385.204-.536-.576-.916-.71-.34-.12-.92.437-1.103.094a1.487 1.487 0 0 1 .617-2.013 1.5 1.5 0 0 1 .713-.174m-10.68 1.55a.96.96 0 1 1 0 1.921.96.96 0 0 1 0-1.92m13.838 0a.96.96 0 1 1 0 1.92.96.96 0 0 1 0-1.92M8.489 11.458c.588.01 1.965 1.157 3.572 1.164 1.607-.007 2.984-1.155 3.572-1.164.196-.003.305.12.305.454 0 .886-.424 2.328-1.563 3.202-.22-.756-1.396-1.366-1.63-1.32q-.011.001-.02.006l-.044.026-.01.008-.03.024q-.018.017-.035.036l-.032.04a1 1 0 0 0-.058.09l-.014.025q-.049.088-.11.19a1 1 0 0 1-.083.116 1.2 1.2 0 0 1-.173.18q-.035.029-.075.058a1.3 1.3 0 0 1-.251-.243 1 1 0 0 1-.076-.107c-.124-.193-.177-.363-.337-.444-.034-.016-.104-.008-.2.022q-.094.03-.216.087-.06.028-.125.063l-.13.074q-.067.04-.136.086a3 3 0 0 0-.135.096 3 3 0 0 0-.26.219 2 2 0 0 0-.12.121 2 2 0 0 0-.106.128l-.002.002a2 2 0 0 0-.09.132l-.001.001a1.2 1.2 0 0 0-.105.212q-.013.036-.024.073c-1.139-.875-1.563-2.317-1.563-3.203 0-.334.109-.457.305-.454m.836 10.354c.824-1.19.766-2.082-.365-3.194-1.13-1.112-1.789-2.738-1.789-2.738s-.246-.945-.806-.858-.97 1.499.202 2.362c1.173.864-.233 1.45-.685.64-.45-.812-1.683-2.896-2.322-3.295s-1.089-.175-.938.647 2.822 2.813 2.562 3.244-1.176-.506-1.176-.506-2.866-2.567-3.49-1.898.473 1.23 2.037 2.16c1.564.932 1.686 1.178 1.464 1.53s-3.675-2.511-4-1.297c-.323 1.214 3.524 1.567 3.287 2.405-.238.839-2.71-1.587-3.216-.642-.506.946 3.49 2.056 3.522 2.064 1.29.33 4.568 1.028 5.713-.624m5.349 0c-.824-1.19-.766-2.082.365-3.194 1.13-1.112 1.789-2.738 1.789-2.738s.246-.945.806-.858.97 1.499-.202 2.362c-1.173.864.233 1.45.685.64.451-.812 1.683-2.896 2.322-3.295s1.089-.175.938.647-2.822 2.813-2.562 3.244 1.176-.506 1.176-.506 2.866-2.567 3.49-1.898-.473 1.23-2.037 2.16c-1.564.932-1.686 1.178-1.464 1.53s3.675-2.511 4-1.297c.323 1.214-3.524 1.567-3.287 2.405.238.839 2.71-1.587 3.216-.642.506.946-3.49 2.056-3.522 2.064-1.29.33-4.568 1.028-5.713-.624"
    />
  </svg>
+{:else if family === "step"}
+  <svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
+    <path
+      d="M22.012 0h1.032v.927H24v.968h-.956V3.78h-1.032V1.896h-1.878v-.97h1.878V0zM2.6 12.371V1.87h.969v10.502h-.97zm10.423.66h10.95v.918h-6.208v9.579h-4.742V13.03zM5.629 3.333v12.356H0v4.51h10.386V8L20.859 8l-.003-4.668-15.227.001z"
+    />
+  </svg>
 {:else}
  <svg class="w-6 h-6 {className}" viewBox="0 0 24 24" fill="currentColor">
    <path
@@ -1577,6 +1577,7 @@ class AppStore {
    // Remove messages after user message (including the user message for image requests
    // since generateImage/editImage will re-add it)
    this.messages = this.messages.slice(0, lastUserIndex);
+    this.updateActiveConversation();

    switch (requestType) {
      case "image-generation":
@@ -42,6 +42,9 @@
    setSelectedChatModel,
    selectedChatModel,
    sendMessage,
+    generateImage,
+    editImage,
+    editingImage,
    messages,
    debugMode,
    toggleDebugMode,
@@ -834,6 +837,52 @@
    if (!model?.tasks) return false;
    return model.tasks.includes("ImageToImage");
  }
+
+  // Route a message to the correct endpoint based on model capabilities.
+  // Image models go to generateImage/editImage; text models go to sendMessage.
+  function routeMessage(
+    content: string,
+    files?: {
+      id: string;
+      name: string;
+      type: string;
+      textContent?: string;
+      preview?: string;
+    }[],
+  ) {
+    const model = selectedChatModel();
+    if (!model) {
+      sendMessage(content, files, null);
+      return;
+    }
+
+    const currentEditImage = editingImage();
+
+    // Image editing mode (explicit edit or attached image with ImageToImage model)
+    if (currentEditImage && content && modelSupportsImageEditing(model)) {
+      editImage(content, currentEditImage.imageDataUrl);
+      return;
+    }
+    if (
+      modelSupportsImageEditing(model) &&
+      files?.length &&
+      files[0].preview &&
+      content
+    ) {
+      editImage(content, files[0].preview);
+      return;
+    }
+
+    // Text-to-image generation
+    if (modelSupportsImageGeneration(model) && content) {
+      generateImage(content);
+      return;
+    }
+
+    // Default: text chat
+    sendMessage(content, files, null);
+  }
+
  let selectedSharding = $state<"Pipeline" | "Tensor">("Pipeline");
  type InstanceMeta = "MlxRing" | "MlxJaccl";

@@ -1527,7 +1576,11 @@
          downloadKind
        ] as Record<string, unknown>;

-        if (downloadKind !== "DownloadOngoing") continue;
+        if (
+          downloadKind !== "DownloadOngoing" &&
+          downloadKind !== "DownloadPending"
+        )
+          continue;
        if (!downloadPayload) continue;

        const downloadModelId = extractModelIdFromDownload(downloadPayload);
@@ -1542,9 +1595,38 @@
          if (downloadModelId !== modelId) continue;
        }

-        isDownloading = true;
+        // For DownloadPending with partial bytes (paused/resumed downloads),
+        // synthesize a progress object from the top-level downloaded/total fields
+        let progress: DownloadProgress | null;
+        if (downloadKind === "DownloadPending") {
+          const pendingDownloaded = getBytes(
+            downloadPayload.downloaded ??
+              downloadPayload.downloaded_bytes ??
+              downloadPayload.downloadedBytes,
+          );
+          const pendingTotal = getBytes(
+            downloadPayload.total ??
+              downloadPayload.total_bytes ??
+              downloadPayload.totalBytes,
+          );
+          if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
+          isDownloading = true;
+          progress = {
+            totalBytes: pendingTotal,
+            downloadedBytes: pendingDownloaded,
+            speed: 0,
+            etaMs: 0,
+            percentage:
+              pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
+            completedFiles: 0,
+            totalFiles: 0,
+            files: [],
+          };
+        } else {
+          isDownloading = true;
+          progress = parseDownloadProgress(downloadPayload);
+        }

-        const progress = parseDownloadProgress(downloadPayload);
        if (progress) {
          // Sum all values across nodes - each node downloads independently
          totalBytes += progress.totalBytes;
@@ -1696,7 +1778,11 @@
          }
        }

-        if (downloadKind !== "DownloadOngoing") continue;
+        if (
+          downloadKind !== "DownloadOngoing" &&
+          downloadKind !== "DownloadPending"
+        )
+          continue;
        if (!downloadPayload) continue;

        // Check if this download is for this instance's model
@@ -1706,9 +1792,37 @@
          downloadModelId &&
          downloadModelId === instanceModelId
        ) {
-          isDownloading = true;
+          // For DownloadPending with partial bytes, synthesize progress
+          let progress: DownloadProgress | null;
+          if (downloadKind === "DownloadPending") {
+            const pendingDownloaded = getBytes(
+              downloadPayload.downloaded ??
+                downloadPayload.downloaded_bytes ??
+                downloadPayload.downloadedBytes,
+            );
+            const pendingTotal = getBytes(
+              downloadPayload.total ??
+                downloadPayload.total_bytes ??
+                downloadPayload.totalBytes,
+            );
+            if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
+            isDownloading = true;
+            progress = {
+              totalBytes: pendingTotal,
+              downloadedBytes: pendingDownloaded,
+              speed: 0,
+              etaMs: 0,
+              percentage:
+                pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
+              completedFiles: 0,
+              totalFiles: 0,
+              files: [],
+            };
+          } else {
+            isDownloading = true;
+            progress = parseDownloadProgress(downloadPayload);
+          }

-          const progress = parseDownloadProgress(downloadPayload);
          if (progress) {
            // Sum all values across nodes - each node downloads independently
            totalBytes += progress.totalBytes;
@@ -2786,7 +2900,7 @@
        // Running model is same or better tier — use it directly
        setSelectedChatModel(bestRunning.id);
        if (!chatStarted) createConversation();
-        sendMessage(content, files);
+        routeMessage(content, files);
        return;
      }
    }
@@ -2803,7 +2917,7 @@
    if (hasRunningInstance(autoModel.id)) {
      setSelectedChatModel(autoModel.id);
      if (!chatStarted) createConversation();
-      sendMessage(content, files);
+      routeMessage(content, files);
      return;
    }

@@ -2956,7 +3070,7 @@
      if (pendingAutoMessage) {
        const msg = pendingAutoMessage;
        pendingAutoMessage = null;
-        sendMessage(msg.content, msg.files);
+        routeMessage(msg.content, msg.files);
      }
      return;
    }
@@ -3035,7 +3149,7 @@
    // Model is selected and running — send directly
    if (model && hasRunningInstance(model)) {
      chatLaunchState = "ready";
-      sendMessage(content, files, null);
+      routeMessage(content, files);
      return;
    }

@@ -117,12 +117,13 @@
              uvLock = builtins.fromTOML (builtins.readFile ./uv.lock);
              mlxPackage = builtins.head (builtins.filter (p: p.name == "mlx" && p.source ? git) uvLock.package);
              uvLockMlxVersion = mlxPackage.version;
+              uvLockMlxRev = builtins.elemAt (builtins.split "#" mlxPackage.source.git) 2;
            in
            {
              metal-toolchain = pkgs.callPackage ./nix/metal-toolchain.nix { };
              mlx = pkgs.callPackage ./nix/mlx.nix {
                inherit (self'.packages) metal-toolchain;
-                inherit uvLockMlxVersion;
+                inherit uvLockMlxVersion uvLockMlxRev;
              };
              default = self'.packages.exo;
            }
@@ -11,6 +11,7 @@
 , fmt
 , python313Packages
 , uvLockMlxVersion
+, uvLockMlxRev
 }:

 assert stdenv.isDarwin;
@@ -41,15 +42,13 @@ let

  mlx = stdenv.mkDerivation rec {
    pname = "mlx";
-    version = let v = "0.30.7.dev20260225+257d5692"; in
-      assert v == uvLockMlxVersion || throw "MLX version mismatch: nix/mlx.nix has ${v} but uv.lock has ${uvLockMlxVersion}. Update both the version and hash in nix/mlx.nix.";
-      v;
+    version = uvLockMlxVersion;
    pyproject = true;

    src = fetchFromGitHub {
      owner = "rltakashige";
      repo = "mlx-jaccl-fix-small-recv";
-      rev = "257d5692fc7af6bba3b8afaeb63c549b7d1e43d5";
+      rev = uvLockMlxRev;
      hash = "sha256-GosFIWxIB48Egb1MqJrR3xhsUsQeWdRk5rV93USY6wQ=";
    };

@@ -25,8 +25,7 @@ dependencies = [
  "openai-harmony>=0.0.8",
  "httpx>=0.28.1",
  "tomlkit>=0.14.0",
-  "pillow>=11.0,<12.0",                        # compatibility with mflux
-  "mflux==0.15.5",
+  "mflux==0.16.9",
  "python-multipart>=0.0.21",
  "msgspec>=0.19.0",
  "zstandard>=0.23.0",
@@ -62,7 +61,7 @@ members = ["rust/exo_pyo3_bindings", "bench"]
 [tool.uv.sources]
 exo_pyo3_bindings = { workspace = true }
 mlx = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git", branch = "address-rdma-gpu-locks", marker = "sys_platform == 'darwin'" }
-mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "fix/float32-logprobs" }
+mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/eval-left-padding-in-batched-rotation" }
 # Uncomment to use local mlx/mlx-lm development versions:
 # mlx = { path = "/Users/Shared/mlx", editable=true }
 # mlx-lm = { path = "/Users/Shared/mlx-lm", editable=true }
@@ -1,6 +1,7 @@
 model_id = "mlx-community/DeepSeek-V3.1-4bit"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 128
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "deepseek"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/DeepSeek-V3.1-8bit"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 128
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "deepseek"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.5-Air-8bit"
 n_layers = 46
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = false
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.5-Air-bf16"
 n_layers = 46
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-4bit"
 n_layers = 91
 hidden_size = 5120
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-6bit"
 n_layers = 91
 hidden_size = 5120
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-8bit-gs32"
 n_layers = 91
 hidden_size = 5120
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-Flash-4bit"
 n_layers = 47
 hidden_size = 2048
+num_key_value_heads = 20
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-Flash-5bit"
 n_layers = 47
 hidden_size = 2048
+num_key_value_heads = 20
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-Flash-6bit"
 n_layers = 47
 hidden_size = 2048
+num_key_value_heads = 20
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-4.7-Flash-8bit"
 n_layers = 47
 hidden_size = 2048
+num_key_value_heads = 20
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-5-8bit-MXFP8"
 n_layers = 78
 hidden_size = 6144
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-5-MXFP4-Q8"
 n_layers = 78
 hidden_size = 6144
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/GLM-5"
 n_layers = 78
 hidden_size = 6144
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "glm"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Kimi-K2-Instruct-4bit"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "kimi"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Kimi-K2-Thinking"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "kimi"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Kimi-K2.5"
 n_layers = 61
 hidden_size = 7168
+num_key_value_heads = 64
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "kimi"
@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-70B-Instruct-HF-4bit"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "4bit"
+base_model = "NVIDIA Llama-3.1-Nemotron-70B-Instruct"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 39688355840
@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-70B-Instruct-HF-8bit"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "8bit"
+base_model = "NVIDIA Llama-3.1-Nemotron-70B-Instruct"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 74964549632
@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-70B-Instruct-HF-bf16"
+n_layers = 80
+hidden_size = 8192
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "bf16"
+base_model = "NVIDIA Llama-3.1-Nemotron-70B-Instruct"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 141107412992
@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-Nano-4B-v1.1-4bit"
+n_layers = 32
+hidden_size = 3072
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "4bit"
+base_model = "NVIDIA Llama-3.1-Nemotron-Nano-4B-v1.1"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 2538706944
@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-Nano-4B-v1.1-8bit"
+n_layers = 32
+hidden_size = 3072
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "8bit"
+base_model = "NVIDIA Llama-3.1-Nemotron-Nano-4B-v1.1"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 4794980352
@@ -0,0 +1,12 @@
+model_id = "mlx-community/Llama-3.1-Nemotron-Nano-4B-v1.1-bf16"
+n_layers = 32
+hidden_size = 3072
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "llama"
+quantization = "bf16"
+base_model = "NVIDIA Llama-3.1-Nemotron-Nano-4B-v1.1"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 9025492992
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.2-1B-Instruct-4bit"
 n_layers = 16
 hidden_size = 2048
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.2-3B-Instruct-4bit"
 n_layers = 28
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.2-3B-Instruct-8bit"
 n_layers = 28
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.3-70B-Instruct-4bit"
 n_layers = 80
 hidden_size = 8192
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Llama-3.3-70B-Instruct-8bit"
 n_layers = 80
 hidden_size = 8192
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"
 n_layers = 80
 hidden_size = 8192
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"
 n_layers = 32
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"
 n_layers = 32
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"
 n_layers = 32
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.1-3bit"
 n_layers = 61
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.1-8bit"
 n_layers = 61
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.5-4bit"
 n_layers = 62
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.5-6bit"
 n_layers = 62
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/MiniMax-M2.5-8bit"
 n_layers = 62
 hidden_size = 3072
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "minimax"
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-4Bit"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "4bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 17775342336
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-5Bit"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "5bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 21721476864
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-6Bit"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "6bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 25667611392
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-8Bit"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "8bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 33559880448
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-BF16"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "bf16"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 63155889408
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-MLX-MXFP4"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "4bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 16788808704
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
+n_layers = 52
+hidden_size = 2688
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "4bit"
+base_model = "NVIDIA Nemotron-3-Nano-30B-A3B"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 19323906944
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-Nano-9B-v2-4bits"
+n_layers = 56
+hidden_size = 4480
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "4bit"
+base_model = "NVIDIA Nemotron-Nano-9B-v2"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 5002791936
@@ -0,0 +1,12 @@
+model_id = "mlx-community/NVIDIA-Nemotron-Nano-9B-v2-6bit"
+n_layers = 56
+hidden_size = 4480
+supports_tensor = true
+tasks = ["TextGeneration"]
+family = "nemotron"
+quantization = "6bit"
+base_model = "NVIDIA Nemotron-Nano-9B-v2"
+capabilities = ["text"]
+
+[storage_size]
+in_bytes = 7224298496
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-0.6B-4bit"
 n_layers = 28
 hidden_size = 1024
+num_key_value_heads = 8
 supports_tensor = false
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-0.6B-8bit"
 n_layers = 28
 hidden_size = 1024
+num_key_value_heads = 8
 supports_tensor = false
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"
 n_layers = 94
 hidden_size = 4096
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"
 n_layers = 94
 hidden_size = 4096
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-30B-A3B-4bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-30B-A3B-8bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"
 n_layers = 62
 hidden_size = 6144
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-8bit"
 n_layers = 62
 hidden_size = 6144
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-4bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-5bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-6bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-8bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Coder-Next-bf16"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"
 n_layers = 48
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-122B-A10B-4bit"
 n_layers = 48
 hidden_size = 3072
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-122B-A10B-6bit"
 n_layers = 48
 hidden_size = 3072
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-122B-A10B-8bit"
 n_layers = 48
 hidden_size = 3072
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-122B-A10B-bf16"
 n_layers = 48
 hidden_size = 3072
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-27B-4bit"
 n_layers = 64
 hidden_size = 5120
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-27B-8bit"
 n_layers = 64
 hidden_size = 5120
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-2B-MLX-8bit"
 n_layers = 24
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-35B-A3B-4bit"
 n_layers = 40
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-35B-A3B-8bit"
 n_layers = 40
 hidden_size = 2048
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-397B-A17B-4bit"
 n_layers = 60
 hidden_size = 4096
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-397B-A17B-6bit"
 n_layers = 60
 hidden_size = 4096
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-397B-A17B-8bit"
 n_layers = 60
 hidden_size = 4096
+num_key_value_heads = 2
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-9B-4bit"
 n_layers = 32
 hidden_size = 4096
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Qwen3.5-9B-8bit"
 n_layers = 32
 hidden_size = 4096
+num_key_value_heads = 4
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "qwen"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Step-3.5-Flash-4bit"
 n_layers = 45
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "step"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Step-3.5-Flash-6bit"
 n_layers = 45
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "step"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/Step-3.5-Flash-8Bit"
 n_layers = 45
 hidden_size = 4096
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "step"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/gpt-oss-120b-MXFP4-Q8"
 n_layers = 36
 hidden_size = 2880
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "gpt-oss"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/gpt-oss-20b-MXFP4-Q8"
 n_layers = 24
 hidden_size = 2880
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "gpt-oss"
@@ -1,6 +1,7 @@
 model_id = "mlx-community/llama-3.3-70b-instruct-fp16"
 n_layers = 80
 hidden_size = 8192
+num_key_value_heads = 8
 supports_tensor = true
 tasks = ["TextGeneration"]
 family = "llama"
@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+"""Fetch num_key_value_heads from HuggingFace config.json and update TOML model cards.
+
+Usage:
+    # Update only cards missing num_key_value_heads
+    uv run python scripts/fetch_kv_heads.py --missing
+
+    # Update all cards (overwrite existing values)
+    uv run python scripts/fetch_kv_heads.py --all
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+import tomlkit
+
+CARDS_DIR = (
+    Path(__file__).resolve().parent.parent / "resources" / "inference_model_cards"
+)
+MAX_WORKERS = 5
+
+
+def fetch_kv_heads(model_id: str) -> int | None:
+    """Fetch num_key_value_heads from HuggingFace config.json."""
+    url = f"https://huggingface.co/{model_id}/raw/main/config.json"
+    try:
+        with urllib.request.urlopen(url, timeout=15) as resp:
+            config = json.loads(resp.read())
+    except Exception as e:
+        print(f"  ERROR fetching {url}: {e}", file=sys.stderr)
+        return None
+
+    for source in [config, config.get("text_config", {})]:
+        if "num_key_value_heads" in source:
+            return int(source["num_key_value_heads"])
+
+    return None
+
+
+def update_toml(path: Path, kv_heads: int) -> bool:
+    """Insert or update num_key_value_heads in a TOML file. Returns True if changed."""
+    content = path.read_text()
+    doc = tomlkit.parse(content)
+
+    if doc.get("num_key_value_heads") == kv_heads:
+        return False
+
+    # Insert after hidden_size if adding for the first time
+    if "num_key_value_heads" not in doc:
+        new_doc = tomlkit.document()
+        for key, value in doc.items():
+            new_doc[key] = value
+            if key == "hidden_size":
+                new_doc["num_key_value_heads"] = kv_heads
+        path.write_text(tomlkit.dumps(new_doc))
+    else:
+        doc["num_key_value_heads"] = kv_heads
+        path.write_text(tomlkit.dumps(doc))
+
+    return True
+
+
+def process_card(path: Path) -> tuple[str, str]:
+    """Fetch and update a single card. Returns (filename, status)."""
+    content = path.read_text()
+    doc = tomlkit.parse(content)
+    model_id = doc.get("model_id")
+    if not model_id:
+        return path.name, "SKIP (no model_id)"
+
+    kv_heads = fetch_kv_heads(str(model_id))
+    if kv_heads is None:
+        return path.name, "FAILED"
+
+    changed = update_toml(path, kv_heads)
+    return path.name, f"{kv_heads} ({'UPDATED' if changed else 'UNCHANGED'})"
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Fetch num_key_value_heads from HuggingFace and update TOML cards."
+    )
+    group = parser.add_mutually_exclusive_group(required=True)
+    group.add_argument(
+        "--all",
+        action="store_true",
+        help="Update all model cards (overwrite existing values)",
+    )
+    group.add_argument(
+        "--missing",
+        action="store_true",
+        help="Only update cards missing num_key_value_heads",
+    )
+    args = parser.parse_args()
+
+    toml_files = sorted(CARDS_DIR.glob("*.toml"))
+    if not toml_files:
+        print(f"No TOML files found in {CARDS_DIR}", file=sys.stderr)
+        sys.exit(1)
+
+    to_process = []
+    skipped = 0
+
+    for path in toml_files:
+        if args.missing and "num_key_value_heads" in path.read_text():
+            skipped += 1
+            continue
+        to_process.append(path)
+
+    updated = 0
+    failed = 0
+
+    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:
+        futures = {pool.submit(process_card, path): path for path in to_process}
+        for future in as_completed(futures):
+            name, status = future.result()
+            print(f"  {name}: {status}")
+            if "UPDATED" in status:
+                updated += 1
+            elif "FAILED" in status:
+                failed += 1
+
+    print(f"\nDone: {updated} updated, {skipped} skipped, {failed} failed")
+
+
+if __name__ == "__main__":
+    main()
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Evan	74b877dbcd	persist node ids in .cache brings back EXO_CACHE_HOME as always ~/.cache/exo/, and store the node id in there. no random copies now!	2026-03-18 11:24:04 +00:00
ciaranbor	a6519ba006	Update mflux to 0.16.9 (#1751 ) Prevents malformed output from Qwen-Image	2026-03-17 16:58:23 +00:00
rltakashige	b713889f73	Fix exo bench again again (#1750 ) Mb premature auto merge	2026-03-17 13:47:24 +00:00
rltakashige	6ee673147d	Fix exo bench prefill and decode tps (#1749 )	2026-03-17 13:36:44 +00:00
ciaranbor	ff4d20eed9	Fix image models through dashboard (#1746 ) ## Motivation Image generation/editing from the dashboard was broken. ChatForm bypassed the parent's model-launch logic, so image requests fell through to text chat. ## Changes - Moved image routing logic from ChatForm.svelte to +page.svelte (routeMessage()) - ChatForm now always delegates to parent via onAutoSend (made required) - Fixed missing updateActiveConversation() call on message retry ## Why It Works All sends now go through the parent's launch-then-route path, so image models get launched before the request is dispatched to the correct endpoint.	2026-03-17 11:22:01 +00:00
Evan Quiney	7ed4639540	use structured concurrency in download coordinator (#1722 ) identical to #1721 but a little safer imo. ## manual testing ran a few downloads, cancelled non-master, cancelled master -- no errors reported.	2026-03-13 14:55:37 +00:00
Mazin Sharaf	29d4165fe2	Add step logo condition to FamilyLogos component (#1676 ) ## Motivation <!-- Why is this change needed? What problem does it solve? --> <!-- If it fixes an open issue, please link to the issue here --> This was to fix a small issue which was that the StepFun logo was not included in the sidebar, and I also noticed it from an open issue: - #1662 ## Changes <!-- Describe what you changed in detail --> I added a condition to the FamilyLogos where if the family is "step" then the logo will be included in the sidebar. ## Test Plan Open exo, go choose a model, and scroll down the sidebar until you see the step logo, which should be there. ### Manual Testing <!-- Hardware: (e.g., MacBook Pro M1 Max 32GB, Mac Mini M2 16GB, connected via Thunderbolt 4) --> <!-- What you did: --> <!-- - --> Check for the step logo in the sidebar.	2026-03-13 13:31:46 +00:00
ecohash-co	12af7c9586	fix: shield macmon cleanup from cancellation to prevent orphaned process (#1714 ) ## Summary Fixes a bug where macmon metrics (GPU usage, temperature, power, RAM) freeze permanently in the dashboard while non-macmon metrics (disk usage) continue updating normally. Root cause: asyncio subprocess pipe transport flow control stall Observed on a 2-node Mac Studio M3 Ultra cluster (Python 3.13.12, anyio 4.11.0, macOS with kqueue) running EXO.app for ~36 hours. Diagnostics confirmed: 1. Only macmon monitoring is stuck — disk metrics from the same InfoGatherer continue updating, proving the Worker, EventRouter, and API pipelines are healthy 2. macmon IS producing data — its stdout pipe is full at exactly 65536 bytes (64KB OS buffer), and macmon is blocked on write at 0% CPU 3. The pipe read-end FD is still open — the exo process holds it, but asyncio isn't reading from it 4. The stale GPU value (0.21) is wrong — should be ~1.0 (matching the other node under identical load) This is consistent with asyncio's subprocess pipe transport getting stuck in flow control: `pause_reading()` is called when the internal buffer exceeds the high-water mark, but `resume_reading()` is never called, permanently deregistering the FD from kqueue. The `receive()` coroutine waits forever for data asyncio will never deliver. ``` $ lsof -p <macmon_pid> macmon 74691 FD=1 PIPE SIZE=65536 ->0x5ae78ecf376ccd0e # full pipe $ lsof -p <exo_pid> \| grep <pipe_id> exo 74689 FD=47 PIPE SIZE=65536 ->0xd129da474c1340f7 # read end held open, not consumed ``` Note: anyio 4.11's `Process.aclose()` already uses `CancelScope(shield=True)` for cleanup during cancellation — this is NOT an election cleanup issue (confirmed by @ciaranbor's testing). Fix: Replace the `async for` iteration with an explicit `receive()` inside `fail_after()`. If macmon produces no output for 10× its configured interval (minimum 30s), `TimeoutError` fires, `open_process` cleanup kills macmon and tears down the stale transport, and the loop restarts with a fresh subprocess and fresh asyncio transport. ## Test plan - [x] All 246 existing tests pass - [ ] Verify macmon restarts after simulated pipe stall (e.g., `SIGSTOP` macmon, wait for timeout, confirm restart and metrics resume) - [ ] Long-running soak test on multi-node cluster to confirm the fix prevents recurrence	2026-03-13 12:54:30 +00:00
ciaranbor	f28b2fd037	Extract mlx revision from uv lock (#1715 ) ## Motivation The MLX version and git revision in nix/mlx.nix were hardcoded and had to be manually kept in sync with uv.lock ## Changes - flake.nix: Extract MLX git rev from uv.lock's source.git URL and pass as uvLockMlxRev - nix/mlx.nix: Use uvLockMlxVersion and uvLockMlxRev instead of hardcoded values; remove version mismatch assertion ## Why It Works uv.lock is already the source of truth — now Nix reads both version and rev from it directly. The pinned fetchFromGitHub hash still guards against unexpected changes.	2026-03-13 12:34:54 +00:00
Mustafa Alp Yılmaz	ea18a62581	fix: guard against ZeroDivisionError in mlx_lm stats (#1707 ) ## Problem Running short completions (like `max_tokens=1` health check probes) can finish so fast that `mlx_lm`'s internal `generation_time` rounds to zero. When that happens, `BatchGenerator.stats()` in `mlx_lm/generate.py` divides `generation_tokens / generation_time` and throws a `ZeroDivisionError`, which kills the runner process. EXO already handles this on its side — lines 289-293 in `batch_generate.py` guard the TPS calculation with `if generation_elapsed > 0`. But the call to `self._exo_gen.stats()` on line 294 goes into mlx_lm's separate timing code, which doesn't have the same guard. Two different timers, only one is protected. In my case, this was triggered by health check probes (content: `"a"`, `max_tokens=1`). The generation completed in sub-microsecond time, `generation_time` was exactly `0`, and the runner crashed. Since the health check command stays in the queue and retries after recovery, it created an infinite crash loop — every ~15 seconds the runner would load the model, get the same health check, and die again. ## Fix Wrap `self._exo_gen.stats()` in a `try/except ZeroDivisionError`. If it throws, set `mlx_stats` to `None` and fall back to `0.0` for `prompt_tps`. The only fields EXO reads from `mlx_stats` are `prompt_tps` and `prompt_time` — losing them on a sub-microsecond generation has no practical impact. ## Traceback ``` File "batch_generate.py", line 294, in step mlx_stats = self._exo_gen.stats() File "mlx_lm/generate.py", line 1224, in stats self._stats.generation_tokens / self._stats.generation_time ZeroDivisionError: division by zero ```	2026-03-12 14:48:36 +00:00
Miguel Cruz	0782d90ec5	fix: show partial download progress on initial dashboard load (#1706 ) ## Summary - Dashboard now shows partial download progress for models that were partially downloaded in a previous session, instead of showing 0% - Both `getModelDownloadStatus()` and `getInstanceDownloadStatus()` now handle `DownloadPending` entries that carry non-zero `downloaded`/`total` bytes Fixes #1042 ## Root cause When exo restarts with partially downloaded models, the `DownloadCoordinator` emits `DownloadPending` events (because `downloaded_this_session` is 0, even though real bytes exist on disk). The main page dashboard only checked for `DownloadOngoing` entries, so these partially downloaded models showed as 0%. The dedicated `/downloads` page already handled this correctly — it renders `DownloadPending` entries with a progress bar when `downloaded > 0`. This fix brings the same behavior to the main page. ## Changes For both `getModelDownloadStatus()` and `getInstanceDownloadStatus()` in `+page.svelte`: - Accept `DownloadPending` in addition to `DownloadOngoing` - For `DownloadPending` entries with `downloaded > 0` or `total > 0`, synthesize a `DownloadProgress` object from the top-level fields (with `speed: 0` and `etaMs: 0` since no active download is in progress) - Skip `DownloadPending` entries where both `downloaded` and `total` are 0 (truly pending, not yet started) ## Test plan - [ ] Partially download a model, quit exo, relaunch — dashboard should show partial progress instead of 0% - [ ] Fully downloaded models still show as complete - [ ] Active downloads still show real-time progress with speed/ETA - [ ] Models never downloaded show as not started (not falsely showing progress) - [ ] Dashboard builds without errors (`cd dashboard && npm run build`) --------- Co-authored-by: Evan <evanev7@gmail.com>	2026-03-12 10:39:34 +00:00
rltakashige	f221a6c85c	Normalise Responses API tool call format (#1704 ) ## Motivation The responses API often does not provide tool args nested under a "function" field. Since we follow the chat completions format of tools in the backend (for MLX chat templates), we need to normalise to this format. ## Test Plan ### Manual Testing Works on n8n! <img width="3442" height="2076" alt="image" src="https://github.com/user-attachments/assets/9e11d679-0102-4d83-9a8e-b0a7a5898708" />	2026-03-11 18:10:12 +00:00
Mustafa Alp Yılmaz	2994b41089	fix: validate num_key_value_heads in tensor sharding placement (#1669 ) ## Problem Models with fewer KV heads than nodes crash during tensor parallelism. For example, Qwen3.5 MoE models have only 2 KV heads — trying to shard across 4 nodes produces empty tensors and a reshape error at runtime. The placement system already validates `hidden_size % num_nodes == 0` but doesn't check KV heads, so it creates configurations that look valid but blow up when the worker tries to split the attention heads. Affected models include Qwen3.5-35B-A3B, Qwen3.5-122B-A10B, Qwen3.5-397B-A17B, Qwen3-Next-80B-A3B, and Qwen3-Coder-Next (all have 2 KV heads). ## Changes Placement validation (`src/exo/master/placement.py`): - Combined KV heads divisibility check with the existing hidden_size filter in a single pass - Cycles where `num_key_value_heads % len(cycle) != 0` are now excluded for tensor sharding - Error message includes both constraints when no valid cycle is found Model card schema (`src/exo/shared/models/model_cards.py`): - Added optional `num_key_value_heads` field to `ModelCard` and `ConfigData` - Extracted from HuggingFace `config.json` (handles both top-level and `text_config` nesting) - Passed through in `fetch_from_hf()` for dynamically fetched cards All 68 inference model cards (`resources/inference_model_cards/.toml`): - Populated `num_key_value_heads` from each model's HuggingFace config Utility script* (`scripts/fetch_kv_heads.py`): - Fetches `num_key_value_heads` from HuggingFace and updates TOML cards - `--missing`: only fills in cards that don't have the field yet - `--all`: re-fetches and overwrites everything - Uses tomlkit for safe TOML editing and ThreadPoolExecutor for parallel fetches ## Behavior - Instance previews no longer show tensor options for models that can't split their KV heads across the cluster size - `place_instance()` rejects with a clear error instead of crash-looping - Pipeline parallelism is unaffected - 2-node tensor still works for 2-KV-head models (2 ÷ 2 = 1) - Field is optional — existing custom cards without it continue to work (validation is skipped when `None`)	2026-03-11 13:46:33 +00:00
Mustafa Alp Yılmaz	38f0c09175	fix: use StreamingDetokenizer in batch generator to fix emoji/UTF-8 corruption (#1691 ) ## Problem Emojis and other multi-byte UTF-8 characters are rendered as `\ufffd` (Unicode Replacement Character) in batch streaming responses. Byte-level BPE tokenizers (like Qwen's) can split multi-byte UTF-8 characters (e.g. a 4-byte emoji) across multiple tokens. The batch generator decodes each token independently with `tokenizer.decode([token_id])`, which produces U+FFFD for partial byte sequences. The sequential generator doesn't have this problem — it uses `stream_generate()` from mlx_lm, which internally uses `StreamingDetokenizer` to buffer incomplete bytes. ## Fix Use `StreamingDetokenizer` in the batch generator, matching the sequential path: - Each `_EngineTask` gets its own `StreamingDetokenizer` instance - `add_token()` buffers tokens, holding back incomplete UTF-8 byte sequences until they form valid characters - `last_segment` returns only complete, valid text - `finalize()` flushes any remaining buffered bytes when generation completes Empty text segments during buffering are harmless — they're already handled correctly by the downstream streaming pipeline. --------- Co-authored-by: Ryuichi Leo Takashige <leo@exolabs.net>	2026-03-10 16:10:22 +00:00
ciaranbor	f36fd56c38	Include power usage in bench responses (#1692 ) ## Motivation Add power/energy usage tracking to /bench API responses. ## Changes - New PowerSampler class that periodically samples sys_power from each node during inference - New PowerUsage / NodePowerStats API types - Integrated into both bench chat completion and bench image generation endpoints ## Why It Works Runs concurrently via anyio.create_task_group, reads from existing state.node_system heartbeat data — no new plumbing needed. Energy = avg_power × elapsed_time. ## Test Plan ### Manual Testing Run a /bench request and verify power_usage appears in the response. ### Automated Testing 7 async tests covering single/multi-node, energy math, dynamic power changes, empty state, and lifecycle.	2026-03-10 16:02:55 +00:00
rltakashige	82c54dd6d6	Add support for Nemotron sharding (#1693 ) ### Automated Testing tested logits match	2026-03-10 15:51:07 +00:00