Fix local model launch preparation handling

api.py keeps growing. it's not tied to the master, so should have it's
own top level folder.

## testing
ci, pytest

CONTRIBUTING.md

@@ -11,9 +11,18 @@ To run EXO from source:
   ```bash
   brew install uv
   ```
-- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
   ```bash
-  brew install macmon
+  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+  rustup toolchain install nightly
+  ```
+- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+  Use the pinned fork revision used by this repo instead of Homebrew `macmon`.
+  ```bash
+  cargo install --git https://github.com/swiftraccoon/macmon \
+    --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
+    macmon \
+    --force
   ```
 
 ```bash

README.md

@@ -95,11 +95,10 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
   ```
 - [uv](https://github.com/astral-sh/uv) (for Python dependency management)
-- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
 - [node](https://github.com/nodejs/node) (for building the dashboard)
 
   ```bash
-  brew install uv macmon node
+  brew install uv node
   ```
 - [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
 
@@ -107,6 +106,17 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
   rustup toolchain install nightly
   ```
+- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+
+  Install the pinned fork revision used by this repo instead of Homebrew `macmon`.
+  Homebrew `macmon 0.6.1` still crashes on Apple M5.
+
+  ```bash
+  cargo install --git https://github.com/swiftraccoon/macmon \
+    --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
+    macmon \
+    --force
+  ```
 
 Clone the repo, build the dashboard, and run exo:
 

dashboard/src/lib/components/ModelCard.svelte

@@ -16,7 +16,9 @@
       perNode?: Array<{
         nodeId: string;
         nodeName: string;
-        progress: DownloadProgress;
+        status: "completed" | "partial" | "pending" | "downloading";
+        percentage: number;
+        progress: DownloadProgress | null;
       }>;
     } | null;
     nodes?: Record<string, NodeInfo>;
@@ -145,10 +147,7 @@
     return `${s}s`;
   }
 
-  const isDownloading = $derived(downloadStatus?.isDownloading ?? false);
-  const progress = $derived(downloadStatus?.progress);
-  const percentage = $derived(progress?.percentage ?? 0);
-  let expandedNodes = $state<Set<string>>(new Set());
+  const perNode = $derived(downloadStatus?.perNode ?? []);
 
   function toggleNodeDetails(nodeId: string): void {
     const next = new Set(expandedNodes);
@@ -587,23 +586,49 @@
       </span>
     </div>
 
-    <!-- Download Status -->
-    {#if isDownloading && progress}
+    <!-- Download Status (per-node) -->
+    {#if perNode.length > 0}
       <div class="mb-2 space-y-1">
-        <div class="flex items-center justify-between text-xs font-mono">
-          <span class="text-blue-400 tracking-wider uppercase">Downloading</span
-          >
-          <span class="text-white/60"
-            >{percentage.toFixed(1)}% &middot; {formatSpeed(progress.speed)}
-            &middot; {formatEta(progress.etaMs)}</span
-          >
-        </div>
-        <div class="h-1 bg-exo-medium-gray/30 rounded overflow-hidden">
-          <div
-            class="h-full bg-blue-500/70 transition-all duration-300"
-            style="width: {percentage}%"
-          ></div>
+        <div
+          class="text-[10px] font-mono text-white/20 tracking-widest uppercase"
+        >
+          Download progress
         </div>
+        {#each perNode as node}
+          <div class="flex items-center gap-2 text-xs font-mono">
+            <span class="text-white/40 w-20 truncate" title={node.nodeId}
+              >{node.nodeName}</span
+            >
+            <div
+              class="flex-1 h-1 bg-exo-medium-gray/30 rounded overflow-hidden"
+            >
+              <div
+                class="h-full transition-all duration-300 {node.status ===
+                'downloading'
+                  ? 'bg-blue-500/70'
+                  : node.status === 'completed'
+                    ? 'bg-exo-yellow/40'
+                    : 'bg-white/20'}"
+                style="width: {node.percentage}%"
+              ></div>
+            </div>
+            <span
+              class="text-right {node.status === 'completed'
+                ? 'text-exo-yellow/60'
+                : node.status === 'downloading'
+                  ? 'text-blue-400/60'
+                  : 'text-white/30'}"
+            >
+              {#if node.status === "downloading" && node.progress}
+                {Math.round(node.percentage)}% {formatSpeed(
+                  node.progress.speed,
+                )}
+              {:else}
+                {node.percentage > 0 ? `${Math.round(node.percentage)}%` : "0%"}
+              {/if}
+            </span>
+          </div>
+        {/each}
       </div>
     {/if}
 
@@ -662,15 +687,7 @@
             {@const allConnections =
               isDebugMode && usedNodes.length > 1
                 ? (() => {
-                    const conns: Array<{
-                      ip: string;
-                      iface: string | null;
-                      from: string;
-                      to: string;
-                      midX: number;
-                      midY: number;
-                      arrow: string;
-                    }> = [];
+                    const conns: Array = [];
                     for (let i = 0; i < usedNodes.length; i++) {
                       for (let j = i + 1; j < usedNodes.length; j++) {
                         const n1 = usedNodes[i];
@@ -682,7 +699,12 @@
                           const toPos = nodePositions[c.to];
                           const arrow =
                             fromPos && toPos ? getArrow(fromPos, toPos) : "→";
-                          conns.push({ ...c, midX, midY, arrow });
+                          conns.push({
+                            ...c,
+                            midX,
+                            midY,
+                            arrow,
+                          });
                         }
                       }
                     }

dashboard/src/routes/+page.svelte

@@ -1535,34 +1535,44 @@
   }
 
   // Helper to get download status for a model (checks all downloads for matching model ID)
-  function getModelDownloadStatus(modelId: string): {
+  type NodeDownloadStatus = {
+    nodeId: string;
+    nodeName: string;
+    status: "completed" | "partial" | "pending" | "downloading";
+    percentage: number;
+    progress: DownloadProgress | null;
+  };
+
+  // Shared helper: collect per-node download status for a model across a set of nodes.
+  // Handles deduplication, entry parsing, and aggregation in one place.
+  function collectDownloadStatus(
+    modelId: string,
+    nodeIds?: string[],
+  ): {
     isDownloading: boolean;
     progress: DownloadProgress | null;
-    perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }>;
+    perNode: NodeDownloadStatus[];
+    failedError: string | null;
   } {
+    const empty = {
+      isDownloading: false,
+      progress: null,
+      perNode: [] as NodeDownloadStatus[],
+      failedError: null,
+    };
+
     if (!downloadsData || Object.keys(downloadsData).length === 0) {
-      return { isDownloading: false, progress: null, perNode: [] };
+      return empty;
     }
 
-    let totalBytes = 0;
-    let downloadedBytes = 0;
-    let totalSpeed = 0;
-    let completedFiles = 0;
-    let totalFiles = 0;
-    let isDownloading = false;
-    const allFiles: DownloadProgress["files"] = [];
-    const perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }> = [];
+    // Deduplicate by nodeId — a node can have multiple entries for the same model
+    // (e.g. PipelineShardMetadata + TensorShardMetadata). Keep the last entry,
+    // which is the most recently applied event.
+    const perNodeMap = new Map<string, NodeDownloadStatus>();
 
-    // Check all nodes for downloads matching this model
+    const nodeIdSet = nodeIds ? new Set(nodeIds) : null;
     for (const [nodeId, nodeDownloads] of Object.entries(downloadsData)) {
+      if (nodeIdSet && !nodeIdSet.has(nodeId)) continue;
       if (!Array.isArray(nodeDownloads)) continue;
 
       for (const downloadWrapped of nodeDownloads) {
@@ -1575,29 +1585,45 @@
         const downloadPayload = (downloadWrapped as Record<string, unknown>)[
           downloadKind
         ] as Record<string, unknown>;
-
-        if (
-          downloadKind !== "DownloadOngoing" &&
-          downloadKind !== "DownloadPending"
-        )
-          continue;
         if (!downloadPayload) continue;
 
         const downloadModelId = extractModelIdFromDownload(downloadPayload);
+        if (!downloadModelId || downloadModelId !== modelId) continue;
 
-        // Match if the model ID contains or equals the requested model
-        // (handles cases like "mlx-community/Meta-Llama..." matching)
-        if (
-          !downloadModelId ||
-          !downloadModelId.includes(modelId.split("/").pop() || modelId)
-        ) {
-          // Try exact match or partial match
-          if (downloadModelId !== modelId) continue;
+        // DownloadFailed — return with any data collected so far
+        if (downloadKind === "DownloadFailed") {
+          return {
+            isDownloading: false,
+            progress: null,
+            perNode: Array.from(perNodeMap.values()),
+            failedError:
+              (downloadPayload.errorMessage as string) ||
+              (downloadPayload.error_message as string) ||
+              "Download failed",
+          };
+        }
+
+        if (
+          downloadKind !== "DownloadOngoing" &&
+          downloadKind !== "DownloadPending" &&
+          downloadKind !== "DownloadCompleted"
+        )
+          continue;
+
+        const nodeName =
+          data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
+
+        if (downloadKind === "DownloadCompleted") {
+          perNodeMap.set(nodeId, {
+            nodeId,
+            nodeName,
+            status: "completed",
+            percentage: 100,
+            progress: null,
+          });
+          continue;
         }
 
-        // For DownloadPending with partial bytes (paused/resumed downloads),
-        // synthesize a progress object from the top-level downloaded/total fields
-        let progress: DownloadProgress | null;
         if (downloadKind === "DownloadPending") {
           const pendingDownloaded = getBytes(
             downloadPayload.downloaded ??
@@ -1610,44 +1636,67 @@
               downloadPayload.totalBytes,
           );
           if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
-          isDownloading = true;
-          progress = {
-            totalBytes: pendingTotal,
-            downloadedBytes: pendingDownloaded,
-            speed: 0,
-            etaMs: 0,
-            percentage:
-              pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
-            completedFiles: 0,
-            totalFiles: 0,
-            files: [],
-          };
-        } else {
-          isDownloading = true;
-          progress = parseDownloadProgress(downloadPayload);
+          const pct =
+            pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0;
+          perNodeMap.set(nodeId, {
+            nodeId,
+            nodeName,
+            status: pendingDownloaded > 0 ? "partial" : "pending",
+            percentage: pct,
+            progress: null,
+          });
+          continue;
         }
 
-        if (progress) {
-          // Sum all values across nodes - each node downloads independently
-          totalBytes += progress.totalBytes;
-          downloadedBytes += progress.downloadedBytes;
-          totalSpeed += progress.speed;
-          completedFiles += progress.completedFiles;
-          totalFiles += progress.totalFiles;
-          allFiles.push(...progress.files);
+        // DownloadOngoing
+        const progress = parseDownloadProgress(downloadPayload);
+        if (
+          !progress ||
+          (progress.downloadedBytes <= 0 && progress.totalBytes <= 0)
+        )
+          continue;
 
-          const nodeName =
-            data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
-          perNode.push({ nodeId, nodeName, progress });
-        }
+        perNodeMap.set(nodeId, {
+          nodeId,
+          nodeName,
+          status: "downloading",
+          percentage: progress.percentage,
+          progress,
+        });
+      }
+    }
+
+    // Aggregate from deduplicated per-node entries
+    const perNode = Array.from(perNodeMap.values());
+    let totalBytes = 0;
+    let downloadedBytes = 0;
+    let totalSpeed = 0;
+    let completedFiles = 0;
+    let totalFiles = 0;
+    let isDownloading = false;
+    const allFiles: DownloadProgress["files"] = [];
+
+    for (const node of perNode) {
+      if (node.status === "downloading" && node.progress) {
+        isDownloading = true;
+        totalBytes += node.progress.totalBytes;
+        downloadedBytes += node.progress.downloadedBytes;
+        totalSpeed += node.progress.speed;
+        completedFiles += node.progress.completedFiles;
+        totalFiles += node.progress.totalFiles;
+        allFiles.push(...node.progress.files);
       }
     }
 
     if (!isDownloading) {
-      return { isDownloading: false, progress: null, perNode: [] };
+      return {
+        isDownloading: false,
+        progress: null,
+        perNode,
+        failedError: null,
+      };
     }
 
-    // ETA = total remaining bytes / total speed across all nodes
     const remainingBytes = totalBytes - downloadedBytes;
     const etaMs = totalSpeed > 0 ? (remainingBytes / totalSpeed) * 1000 : 0;
 
@@ -1664,9 +1713,21 @@
         files: allFiles,
       },
       perNode,
+      failedError: null,
     };
   }
 
+  function getModelDownloadStatus(
+    modelId: string,
+    nodeIds?: string[],
+  ): {
+    isDownloading: boolean;
+    progress: DownloadProgress | null;
+    perNode: NodeDownloadStatus[];
+  } {
+    return collectDownloadStatus(modelId, nodeIds);
+  }
+
   // Helper to get download status for an instance
   function getInstanceDownloadStatus(
     instanceId: string,
@@ -1677,26 +1738,9 @@
     errorMessage: string | null;
     progress: DownloadProgress | null;
     statusText: string;
-    perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }>;
+    perNode: NodeDownloadStatus[];
   } {
-    if (!downloadsData || Object.keys(downloadsData).length === 0) {
-      // No download data yet — defer to runner status instead of assuming RUNNING
-      const statusInfo = deriveInstanceStatus(instanceWrapped);
-      return {
-        isDownloading: false,
-        isFailed: false,
-        errorMessage: null,
-        progress: null,
-        statusText: statusInfo.statusText,
-        perNode: [],
-      };
-    }
-
-    // Unwrap the instance
+    // Unwrap the instance to get shard assignments
     const [instanceTag, instance] = getTagged(instanceWrapped);
     if (!instance || typeof instance !== "object") {
       return {
@@ -1716,132 +1760,9 @@
         modelId?: string;
       };
     };
-    const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
-    const runnerToShard = inst.shardAssignments?.runnerToShard || {};
     const instanceModelId = inst.shardAssignments?.modelId;
 
-    // Build reverse mapping: runnerId -> nodeId
-    const runnerToNode: Record<string, string> = {};
-    for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) {
-      runnerToNode[runnerId] = nodeId;
-    }
-
-    let totalBytes = 0;
-    let downloadedBytes = 0;
-    let totalSpeed = 0;
-    let completedFiles = 0;
-    let totalFiles = 0;
-    let isDownloading = false;
-    const allFiles: DownloadProgress["files"] = [];
-    const perNode: Array<{
-      nodeId: string;
-      nodeName: string;
-      progress: DownloadProgress;
-    }> = [];
-
-    // Check downloads for nodes that are part of this instance
-    for (const runnerId of Object.keys(runnerToShard)) {
-      const nodeId = runnerToNode[runnerId];
-      if (!nodeId) continue;
-
-      const nodeDownloads = downloadsData[nodeId];
-      if (!Array.isArray(nodeDownloads)) continue;
-
-      for (const downloadWrapped of nodeDownloads) {
-        if (!downloadWrapped || typeof downloadWrapped !== "object") continue;
-
-        const keys = Object.keys(downloadWrapped as Record<string, unknown>);
-        if (keys.length !== 1) continue;
-
-        const downloadKind = keys[0];
-        const downloadPayload = (downloadWrapped as Record<string, unknown>)[
-          downloadKind
-        ] as Record<string, unknown>;
-
-        // Handle DownloadFailed - return immediately with error info
-        if (downloadKind === "DownloadFailed") {
-          const downloadModelId = extractModelIdFromDownload(downloadPayload);
-          if (
-            instanceModelId &&
-            downloadModelId &&
-            downloadModelId === instanceModelId
-          ) {
-            return {
-              isDownloading: false,
-              isFailed: true,
-              errorMessage:
-                (downloadPayload.errorMessage as string) || "Download failed",
-              progress: null,
-              statusText: "FAILED",
-              perNode: [],
-            };
-          }
-        }
-
-        if (
-          downloadKind !== "DownloadOngoing" &&
-          downloadKind !== "DownloadPending"
-        )
-          continue;
-        if (!downloadPayload) continue;
-
-        // Check if this download is for this instance's model
-        const downloadModelId = extractModelIdFromDownload(downloadPayload);
-        if (
-          instanceModelId &&
-          downloadModelId &&
-          downloadModelId === instanceModelId
-        ) {
-          // For DownloadPending with partial bytes, synthesize progress
-          let progress: DownloadProgress | null;
-          if (downloadKind === "DownloadPending") {
-            const pendingDownloaded = getBytes(
-              downloadPayload.downloaded ??
-                downloadPayload.downloaded_bytes ??
-                downloadPayload.downloadedBytes,
-            );
-            const pendingTotal = getBytes(
-              downloadPayload.total ??
-                downloadPayload.total_bytes ??
-                downloadPayload.totalBytes,
-            );
-            if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
-            isDownloading = true;
-            progress = {
-              totalBytes: pendingTotal,
-              downloadedBytes: pendingDownloaded,
-              speed: 0,
-              etaMs: 0,
-              percentage:
-                pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
-              completedFiles: 0,
-              totalFiles: 0,
-              files: [],
-            };
-          } else {
-            isDownloading = true;
-            progress = parseDownloadProgress(downloadPayload);
-          }
-
-          if (progress) {
-            // Sum all values across nodes - each node downloads independently
-            totalBytes += progress.totalBytes;
-            downloadedBytes += progress.downloadedBytes;
-            totalSpeed += progress.speed;
-            completedFiles += progress.completedFiles;
-            totalFiles += progress.totalFiles;
-            allFiles.push(...progress.files);
-
-            const nodeName =
-              data?.nodes?.[nodeId]?.friendly_name ?? nodeId.slice(0, 8);
-            perNode.push({ nodeId, nodeName, progress });
-          }
-        }
-      }
-    }
-
-    if (!isDownloading) {
-      // Check runner status for other states
+    if (!instanceModelId) {
       const statusInfo = deriveInstanceStatus(instanceWrapped);
       return {
         isDownloading: false,
@@ -1853,26 +1774,49 @@
       };
     }
 
-    // ETA = total remaining bytes / total speed across all nodes
-    const remainingBytes = totalBytes - downloadedBytes;
-    const etaMs = totalSpeed > 0 ? (remainingBytes / totalSpeed) * 1000 : 0;
+    // Get node IDs assigned to this instance
+    const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
+    const runnerToShard = inst.shardAssignments?.runnerToShard || {};
+    const runnerToNode: Record<string, string> = {};
+    for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) {
+      runnerToNode[runnerId] = nodeId;
+    }
+    const instanceNodeIds = Object.keys(runnerToShard)
+      .map((runnerId) => runnerToNode[runnerId])
+      .filter(Boolean);
+
+    const result = collectDownloadStatus(instanceModelId, instanceNodeIds);
+
+    if (result.failedError) {
+      return {
+        isDownloading: false,
+        isFailed: true,
+        errorMessage: result.failedError,
+        progress: null,
+        statusText: "FAILED",
+        perNode: [],
+      };
+    }
+
+    if (!result.isDownloading) {
+      const statusInfo = deriveInstanceStatus(instanceWrapped);
+      return {
+        isDownloading: false,
+        isFailed: statusInfo.statusText === "FAILED",
+        errorMessage: null,
+        progress: null,
+        statusText: statusInfo.statusText,
+        perNode: result.perNode,
+      };
+    }
 
     return {
       isDownloading: true,
       isFailed: false,
       errorMessage: null,
-      progress: {
-        totalBytes,
-        downloadedBytes,
-        speed: totalSpeed,
-        etaMs,
-        percentage: totalBytes > 0 ? (downloadedBytes / totalBytes) * 100 : 0,
-        completedFiles,
-        totalFiles,
-        files: allFiles,
-      },
+      progress: result.progress,
       statusText: "DOWNLOADING",
-      perNode,
+      perNode: result.perNode,
     };
   }
 
@@ -5369,10 +5313,10 @@
                             <div
                               class="mt-2 space-y-2 max-h-48 overflow-y-auto pr-1"
                             >
-                              {#each downloadInfo.perNode as nodeProg}
+                              {#each downloadInfo.perNode.filter((n) => n.status === "downloading" && n.progress) as nodeProg}
                                 {@const nodePercent = Math.min(
                                   100,
-                                  Math.max(0, nodeProg.progress.percentage),
+                                  Math.max(0, nodeProg.percentage),
                                 )}
                                 {@const isExpanded =
                                   instanceDownloadExpandedNodes.has(
@@ -5428,15 +5372,17 @@
                                     >
                                       <span
                                         >{formatBytes(
-                                          nodeProg.progress.downloadedBytes,
+                                          nodeProg.progress?.downloadedBytes ??
+                                            0,
                                         )} / {formatBytes(
-                                          nodeProg.progress.totalBytes,
+                                          nodeProg.progress?.totalBytes ?? 0,
                                         )}</span
                                       >
                                       <span
-                                        >{formatSpeed(nodeProg.progress.speed)} •
-                                        ETA {formatEta(
-                                          nodeProg.progress.etaMs,
+                                        >{formatSpeed(
+                                          nodeProg.progress?.speed ?? 0,
+                                        )} • ETA {formatEta(
+                                          nodeProg.progress?.etaMs ?? 0,
                                         )}</span
                                       >
                                     </div>
@@ -5444,14 +5390,14 @@
 
                                   {#if isExpanded}
                                     <div class="mt-2 space-y-1.5">
-                                      {#if nodeProg.progress.files.length === 0}
+                                      {#if nodeProg.progress?.files ?? [].length === 0}
                                         <div
                                           class="text-[11px] font-mono text-exo-light-gray/70"
                                         >
                                           No file details reported.
                                         </div>
                                       {:else}
-                                        {#each nodeProg.progress.files as f}
+                                        {#each nodeProg.progress?.files ?? [] as f}
                                           {@const filePercent = Math.min(
                                             100,
                                             Math.max(0, f.percentage ?? 0),
@@ -5927,12 +5873,15 @@
                 )}
                 {@const allPreviews = filteredPreviews()}
                 {#if selectedModel && allPreviews.length > 0}
-                  {@const downloadStatus = getModelDownloadStatus(
-                    selectedModel.id,
-                  )}
                   {@const tags = modelTags()[selectedModel.id] || []}
                   <div class="space-y-3">
                     {#each allPreviews as apiPreview, i}
+                      {@const downloadStatus = getModelDownloadStatus(
+                        selectedModel.id,
+                        apiPreview.memory_delta_by_node
+                          ? Object.keys(apiPreview.memory_delta_by_node)
+                          : undefined,
+                      )}
                       <div
                         role="group"
                         onmouseenter={() => {
@@ -6503,10 +6452,10 @@
                               <div
                                 class="mt-2 space-y-2 max-h-48 overflow-y-auto pr-1"
                               >
-                                {#each downloadInfo.perNode as nodeProg}
+                                {#each downloadInfo.perNode.filter((n) => n.status === "downloading" && n.progress) as nodeProg}
                                   {@const nodePercent = Math.min(
                                     100,
-                                    Math.max(0, nodeProg.progress.percentage),
+                                    Math.max(0, nodeProg.percentage),
                                   )}
                                   {@const isExpanded =
                                     instanceDownloadExpandedNodes.has(
@@ -6565,16 +6514,17 @@
                                       >
                                         <span
                                           >{formatBytes(
-                                            nodeProg.progress.downloadedBytes,
+                                            nodeProg.progress
+                                              ?.downloadedBytes ?? 0,
                                           )} / {formatBytes(
-                                            nodeProg.progress.totalBytes,
+                                            nodeProg.progress?.totalBytes ?? 0,
                                           )}</span
                                         >
                                         <span
                                           >{formatSpeed(
-                                            nodeProg.progress.speed,
+                                            nodeProg.progress?.speed ?? 0,
                                           )} • ETA {formatEta(
-                                            nodeProg.progress.etaMs,
+                                            nodeProg.progress?.etaMs ?? 0,
                                           )}</span
                                         >
                                       </div>
@@ -6582,14 +6532,14 @@
 
                                     {#if isExpanded}
                                       <div class="mt-2 space-y-1.5">
-                                        {#if nodeProg.progress.files.length === 0}
+                                        {#if nodeProg.progress?.files ?? [].length === 0}
                                           <div
                                             class="text-[11px] font-mono text-exo-light-gray/70"
                                           >
                                             No file details reported.
                                           </div>
                                         {:else}
-                                          {#each nodeProg.progress.files as f}
+                                          {#each nodeProg.progress?.files ?? [] as f}
                                             {@const filePercent = Math.min(
                                               100,
                                               Math.max(0, f.percentage ?? 0),

flake.nix

@@ -72,7 +72,7 @@
       ];
 
       perSystem =
-        { config, self', inputs', pkgs, lib, system, ... }:
+        { config, self', pkgs, lib, system, ... }:
         let
           # Use pinned nixpkgs for swift-format (swift is broken on x86_64-linux in newer nixpkgs)
           pkgsSwift = import inputs.nixpkgs-swift { inherit system; };
@@ -84,6 +84,17 @@
             config.allowUnfreePredicate = pkg: (pkg.pname or "") == "metal-toolchain";
             overlays = [
               (import ./nix/apple-sdk-overlay.nix)
+              (final: prev: {
+                macmon = prev.macmon.overrideAttrs (_: {
+                  version = "git";
+                  src = final.fetchFromGitHub {
+                    owner = "swiftraccoon";
+                    repo = "macmon";
+                    rev = "9154d234f763fbeffdcb4135d0bbbaf80609699b";
+                    hash = "sha256-CwhilKNbs5XL9/tF5DMwyPBlE/hpmjGNTuxQ36sM50M=";
+                  };
+                });
+              })
             ];
           };
           treefmt = {

packaging/pyinstaller/exo.spec

@@ -71,7 +71,9 @@ MACMON_PATH = shutil.which("macmon")
 if MACMON_PATH is None:
     raise SystemExit(
         "macmon binary not found in PATH. "
-        "Install it via: brew install macmon"
+        "Install the pinned fork used by exo via: "
+        "cargo install --git https://github.com/swiftraccoon/macmon "
+        "--rev 9154d234f763fbeffdcb4135d0bbbaf80609699b macmon --force"
     )
 
 BINARIES: list[tuple[str, str]] = [
@@ -120,4 +122,3 @@ coll = COLLECT(
     upx_exclude=[],
     name="exo",
 )
-

src/exo/api/adapters/chat_completions.py

@@ -4,7 +4,7 @@ import time
 from collections.abc import AsyncGenerator
 from typing import Any
 
-from exo.shared.types.api import (
+from exo.api.types import (
     ChatCompletionChoice,
     ChatCompletionMessage,
     ChatCompletionMessageText,

src/exo/api/adapters/claude.py

@@ -5,14 +5,8 @@ import re
 from collections.abc import AsyncGenerator
 from typing import Any
 
-from exo.shared.types.api import FinishReason, Usage
-from exo.shared.types.chunks import (
-    ErrorChunk,
-    PrefillProgressChunk,
-    TokenChunk,
-    ToolCallChunk,
-)
-from exo.shared.types.claude_api import (
+from exo.api.types import FinishReason, Usage
+from exo.api.types.claude_api import (
     ClaudeContentBlock,
     ClaudeContentBlockDeltaEvent,
     ClaudeContentBlockStartEvent,
@@ -35,6 +29,12 @@ from exo.shared.types.claude_api import (
     ClaudeToolUseBlock,
     ClaudeUsage,
 )
+from exo.shared.types.chunks import (
+    ErrorChunk,
+    PrefillProgressChunk,
+    TokenChunk,
+    ToolCallChunk,
+)
 from exo.shared.types.common import CommandId
 from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
 

src/exo/api/adapters/ollama.py

@@ -4,14 +4,7 @@ import json
 from collections.abc import AsyncGenerator
 from typing import Any
 
-from exo.shared.types.chunks import (
-    ErrorChunk,
-    PrefillProgressChunk,
-    TokenChunk,
-    ToolCallChunk,
-)
-from exo.shared.types.common import CommandId
-from exo.shared.types.ollama_api import (
+from exo.api.types.ollama_api import (
     OllamaChatRequest,
     OllamaChatResponse,
     OllamaDoneReason,
@@ -21,6 +14,13 @@ from exo.shared.types.ollama_api import (
     OllamaToolCall,
     OllamaToolFunction,
 )
+from exo.shared.types.chunks import (
+    ErrorChunk,
+    PrefillProgressChunk,
+    TokenChunk,
+    ToolCallChunk,
+)
+from exo.shared.types.common import CommandId
 from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
 
 

src/exo/api/adapters/responses.py

@@ -4,15 +4,8 @@ from collections.abc import AsyncGenerator
 from itertools import count
 from typing import Any
 
-from exo.shared.types.api import Usage
-from exo.shared.types.chunks import (
-    ErrorChunk,
-    PrefillProgressChunk,
-    TokenChunk,
-    ToolCallChunk,
-)
-from exo.shared.types.common import CommandId
-from exo.shared.types.openai_responses import (
+from exo.api.types import Usage
+from exo.api.types.openai_responses import (
     FunctionCallInputItem,
     ResponseCompletedEvent,
     ResponseContentPart,
@@ -42,6 +35,13 @@ from exo.shared.types.openai_responses import (
     ResponseTextDoneEvent,
     ResponseUsage,
 )
+from exo.shared.types.chunks import (
+    ErrorChunk,
+    PrefillProgressChunk,
+    TokenChunk,
+    ToolCallChunk,
+)
+from exo.shared.types.common import CommandId
 from exo.shared.types.text_generation import (
     InputMessage,
     TextGenerationTaskParams,

src/exo/api/main.py

@@ -21,17 +21,17 @@ from hypercorn.config import Config
 from hypercorn.typing import ASGIFramework
 from loguru import logger
 
-from exo.master.adapters.chat_completions import (
+from exo.api.adapters.chat_completions import (
     chat_request_to_text_generation,
     collect_chat_response,
     generate_chat_stream,
 )
-from exo.master.adapters.claude import (
+from exo.api.adapters.claude import (
     claude_request_to_text_generation,
     collect_claude_response,
     generate_claude_stream,
 )
-from exo.master.adapters.ollama import (
+from exo.api.adapters.ollama import (
     collect_ollama_chat_response,
     collect_ollama_generate_response,
     generate_ollama_chat_stream,
@@ -39,34 +39,12 @@ from exo.master.adapters.ollama import (
     ollama_generate_request_to_text_generation,
     ollama_request_to_text_generation,
 )
-from exo.master.adapters.responses import (
+from exo.api.adapters.responses import (
     collect_responses_response,
     generate_responses_stream,
     responses_request_to_text_generation,
 )
-from exo.master.event_log import DiskEventLog
-from exo.master.image_store import ImageStore
-from exo.master.placement import place_instance as get_instance_placements
-from exo.shared.apply import apply
-from exo.shared.constants import (
-    DASHBOARD_DIR,
-    EXO_CACHE_HOME,
-    EXO_EVENT_LOG_DIR,
-    EXO_IMAGE_CACHE_DIR,
-    EXO_MAX_CHUNK_SIZE,
-    EXO_TRACING_CACHE_DIR,
-)
-from exo.shared.election import ElectionMessage
-from exo.shared.logging import InterceptLogger
-from exo.shared.models.model_cards import (
-    ModelCard,
-    ModelId,
-    delete_custom_card,
-    get_model_cards,
-    is_custom_card,
-)
-from exo.shared.tracing import TraceEvent, compute_stats, export_trace, load_trace_file
-from exo.shared.types.api import (
+from exo.api.types import (
     AddCustomModelParams,
     AdvancedImageParams,
     BenchChatCompletionRequest,
@@ -114,6 +92,48 @@ from exo.shared.types.api import (
     TraceStatsResponse,
     normalize_image_size,
 )
+from exo.api.types.claude_api import (
+    ClaudeMessagesRequest,
+    ClaudeMessagesResponse,
+)
+from exo.api.types.ollama_api import (
+    OllamaChatRequest,
+    OllamaChatResponse,
+    OllamaGenerateRequest,
+    OllamaGenerateResponse,
+    OllamaModelDetails,
+    OllamaModelTag,
+    OllamaPsModel,
+    OllamaPsResponse,
+    OllamaShowRequest,
+    OllamaShowResponse,
+    OllamaTagsResponse,
+)
+from exo.api.types.openai_responses import (
+    ResponsesRequest,
+    ResponsesResponse,
+)
+from exo.master.image_store import ImageStore
+from exo.master.placement import place_instance as get_instance_placements
+from exo.shared.apply import apply
+from exo.shared.constants import (
+    DASHBOARD_DIR,
+    EXO_CACHE_HOME,
+    EXO_EVENT_LOG_DIR,
+    EXO_IMAGE_CACHE_DIR,
+    EXO_MAX_CHUNK_SIZE,
+    EXO_TRACING_CACHE_DIR,
+)
+from exo.shared.election import ElectionMessage
+from exo.shared.logging import InterceptLogger
+from exo.shared.models.model_cards import (
+    ModelCard,
+    ModelId,
+    delete_custom_card,
+    get_model_cards,
+    is_custom_card,
+)
+from exo.shared.tracing import TraceEvent, compute_stats, export_trace, load_trace_file
 from exo.shared.types.chunks import (
     ErrorChunk,
     ImageChunk,
@@ -122,10 +142,6 @@ from exo.shared.types.chunks import (
     TokenChunk,
     ToolCallChunk,
 )
-from exo.shared.types.claude_api import (
-    ClaudeMessagesRequest,
-    ClaudeMessagesResponse,
-)
 from exo.shared.types.commands import (
     Command,
     CreateInstance,
@@ -151,29 +167,13 @@ from exo.shared.types.events import (
     TracesMerged,
 )
 from exo.shared.types.memory import Memory
-from exo.shared.types.ollama_api import (
-    OllamaChatRequest,
-    OllamaChatResponse,
-    OllamaGenerateRequest,
-    OllamaGenerateResponse,
-    OllamaModelDetails,
-    OllamaModelTag,
-    OllamaPsModel,
-    OllamaPsResponse,
-    OllamaShowRequest,
-    OllamaShowResponse,
-    OllamaTagsResponse,
-)
-from exo.shared.types.openai_responses import (
-    ResponsesRequest,
-    ResponsesResponse,
-)
 from exo.shared.types.state import State
 from exo.shared.types.worker.downloads import DownloadCompleted
 from exo.shared.types.worker.instances import Instance, InstanceId, InstanceMeta
 from exo.shared.types.worker.shards import Sharding
 from exo.utils.banner import print_startup_banner
 from exo.utils.channels import Receiver, Sender, channel
+from exo.utils.disk_event_log import DiskEventLog
 from exo.utils.power_sampler import PowerSampler
 from exo.utils.task_group import TaskGroup
 

src/exo/api/tests/test_api_error_handling.py

@@ -4,10 +4,11 @@ from typing import Any
 from fastapi import FastAPI, HTTPException
 from fastapi.testclient import TestClient
 
+from exo.api.main import API
+
 
 def test_http_exception_handler_formats_openai_style() -> None:
     """Test that HTTPException is converted to OpenAI-style error format."""
-    from exo.master.api import API
 
     app = FastAPI()
 

src/exo/api/tests/test_cancel_command.py

@@ -5,12 +5,12 @@ from unittest.mock import AsyncMock, MagicMock
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
+from exo.api.main import API
 from exo.shared.types.common import CommandId
 
 
 def _make_api() -> Any:
     """Create a minimal API instance with cancel route and error handler."""
-    from exo.master.api import API
 
     app = FastAPI()
     api = object.__new__(API)

src/exo/api/tests/test_claude_api.py

@@ -3,11 +3,11 @@
 import pydantic
 import pytest
 
-from exo.master.adapters.claude import (
+from exo.api.adapters.claude import (
     claude_request_to_text_generation,
     finish_reason_to_claude_stop_reason,
 )
-from exo.shared.types.claude_api import (
+from exo.api.types.claude_api import (
     ClaudeMessage,
     ClaudeMessagesRequest,
     ClaudeTextBlock,

src/exo/api/tests/test_claude_tool_use.py

@@ -4,12 +4,12 @@ import json
 from collections.abc import AsyncGenerator
 from typing import Any, cast
 
-from exo.master.adapters.claude import (
+from exo.api.adapters.claude import (
     ClaudeMessagesResponse,
     collect_claude_response,
     generate_claude_stream,
 )
-from exo.shared.types.api import ToolCallItem
+from exo.api.types import ToolCallItem
 from exo.shared.types.chunks import ErrorChunk, TokenChunk, ToolCallChunk
 from exo.shared.types.common import CommandId, ModelId
 

src/exo/api/tests/test_openai_responses_api.py

@@ -7,11 +7,11 @@ The responses adapter converts it to TextGenerationTaskParams for the pipeline.
 import pydantic
 import pytest
 
-from exo.shared.types.common import ModelId
-from exo.shared.types.openai_responses import (
+from exo.api.types.openai_responses import (
     ResponseInputMessage,
     ResponsesRequest,
 )
+from exo.shared.types.common import ModelId
 
 
 class TestResponsesRequestValidation:

src/exo/api/types/__init__.py

@@ -0,0 +1,57 @@
+from .api import AddCustomModelParams as AddCustomModelParams
+from .api import AdvancedImageParams as AdvancedImageParams
+from .api import BenchChatCompletionRequest as BenchChatCompletionRequest
+from .api import BenchChatCompletionResponse as BenchChatCompletionResponse
+from .api import BenchImageGenerationResponse as BenchImageGenerationResponse
+from .api import BenchImageGenerationTaskParams as BenchImageGenerationTaskParams
+from .api import CancelCommandResponse as CancelCommandResponse
+from .api import ChatCompletionChoice as ChatCompletionChoice
+from .api import ChatCompletionMessage as ChatCompletionMessage
+from .api import ChatCompletionMessageText as ChatCompletionMessageText
+from .api import ChatCompletionRequest as ChatCompletionRequest
+from .api import ChatCompletionResponse as ChatCompletionResponse
+from .api import CompletionTokensDetails as CompletionTokensDetails
+from .api import CreateInstanceParams as CreateInstanceParams
+from .api import CreateInstanceResponse as CreateInstanceResponse
+from .api import DeleteDownloadResponse as DeleteDownloadResponse
+from .api import DeleteInstanceResponse as DeleteInstanceResponse
+from .api import DeleteTracesRequest as DeleteTracesRequest
+from .api import DeleteTracesResponse as DeleteTracesResponse
+from .api import ErrorInfo as ErrorInfo
+from .api import ErrorResponse as ErrorResponse
+from .api import FinishReason as FinishReason
+from .api import GenerationStats as GenerationStats
+from .api import HuggingFaceSearchResult as HuggingFaceSearchResult
+from .api import ImageData as ImageData
+from .api import ImageEditsTaskParams as ImageEditsTaskParams
+from .api import ImageGenerationResponse as ImageGenerationResponse
+from .api import ImageGenerationStats as ImageGenerationStats
+from .api import ImageGenerationTaskParams as ImageGenerationTaskParams
+from .api import ImageListItem as ImageListItem
+from .api import ImageListResponse as ImageListResponse
+from .api import ImageSize as ImageSize
+from .api import Logprobs as Logprobs
+from .api import LogprobsContentItem as LogprobsContentItem
+from .api import ModelList as ModelList
+from .api import ModelListModel as ModelListModel
+from .api import NodePowerStats as NodePowerStats
+from .api import PlaceInstanceParams as PlaceInstanceParams
+from .api import PlacementPreview as PlacementPreview
+from .api import PlacementPreviewResponse as PlacementPreviewResponse
+from .api import PowerUsage as PowerUsage
+from .api import PromptTokensDetails as PromptTokensDetails
+from .api import StartDownloadParams as StartDownloadParams
+from .api import StartDownloadResponse as StartDownloadResponse
+from .api import StreamingChoiceResponse as StreamingChoiceResponse
+from .api import ToolCall as ToolCall
+from .api import ToolCallItem as ToolCallItem
+from .api import TopLogprobItem as TopLogprobItem
+from .api import TraceCategoryStats as TraceCategoryStats
+from .api import TraceEventResponse as TraceEventResponse
+from .api import TraceListItem as TraceListItem
+from .api import TraceListResponse as TraceListResponse
+from .api import TraceRankStats as TraceRankStats
+from .api import TraceResponse as TraceResponse
+from .api import TraceStatsResponse as TraceStatsResponse
+from .api import Usage as Usage
+from .api import normalize_image_size as normalize_image_size

src/exo/download/coordinator.py

@@ -7,6 +7,7 @@ from loguru import logger
 from exo.download.download_utils import (
     RepoDownloadProgress,
     delete_model,
+    is_model_directory_complete,
     map_repo_download_progress_to_download_progress_data,
     resolve_model_in_path,
 )
@@ -168,6 +169,23 @@ class DownloadCoordinator:
             )
             return
 
+        local_model_dir = EXO_MODELS_DIR / model_id.normalize()
+        if local_model_dir.is_dir() and is_model_directory_complete(local_model_dir):
+            logger.info(
+                f"DownloadCoordinator: Model {model_id} already complete at {local_model_dir}"
+            )
+            completed = DownloadCompleted(
+                shard_metadata=shard,
+                node_id=self.node_id,
+                total=shard.model_card.storage_size,
+                model_directory=str(local_model_dir),
+            )
+            self.download_status[model_id] = completed
+            await self.event_sender.send(
+                NodeDownloadProgress(download_progress=completed)
+            )
+            return
+
         # Emit pending status
         progress = DownloadPending(
             shard_metadata=shard,

src/exo/download/download_utils.py

@@ -239,6 +239,17 @@ def _scan_model_directory(
 
 def is_model_directory_complete(model_dir: Path) -> bool:
     """Check if a model directory contains all required weight files."""
+    index_files = list(model_dir.glob("**/*.safetensors.index.json"))
+    if not index_files:
+        return False
+
+    for index_file in index_files:
+        try:
+            ModelSafetensorsIndex.model_validate_json(index_file.read_text())
+        except Exception:
+            logger.warning(f"Failed to parse model index {index_file}")
+            return False
+
     file_list = _scan_model_directory(model_dir, recursive=True)
     return file_list is not None and all(f.size is not None for f in file_list)
 

src/exo/download/tests/test_download_verification.py

@@ -14,6 +14,7 @@ from pydantic import TypeAdapter
 from exo.download.download_utils import (
     delete_model,
     fetch_file_list_with_cache,
+    is_model_directory_complete,
 )
 from exo.shared.types.common import ModelId
 from exo.shared.types.memory import Memory
@@ -324,6 +325,69 @@ class TestModelDeletion:
             assert result is False
 
 
+class TestModelDirectoryComplete:
+    """Tests for local completeness checks used to skip download probing."""
+
+    async def test_returns_false_when_only_partial_weight_exists(
+        self, model_id: ModelId, tmp_path: Path
+    ) -> None:
+        import json
+
+        model_dir = tmp_path / model_id.normalize()
+        await aios.makedirs(model_dir, exist_ok=True)
+
+        async with aiofiles.open(model_dir / "model.safetensors.index.json", "w") as f:
+            await f.write(
+                json.dumps(
+                    {
+                        "metadata": {"total_size": 256},
+                        "weight_map": {"model.layers.0.weight": "model.safetensors"},
+                    }
+                )
+            )
+        async with aiofiles.open(model_dir / "model.safetensors.partial", "wb") as f:
+            await f.write(b"x" * 128)
+
+        assert is_model_directory_complete(model_dir) is False
+
+    async def test_returns_true_when_final_weight_exists_even_with_stale_partial(
+        self, model_id: ModelId, tmp_path: Path
+    ) -> None:
+        import json
+
+        model_dir = tmp_path / model_id.normalize()
+        await aios.makedirs(model_dir, exist_ok=True)
+
+        async with aiofiles.open(model_dir / "model.safetensors.index.json", "w") as f:
+            await f.write(
+                json.dumps(
+                    {
+                        "metadata": {"total_size": 256},
+                        "weight_map": {"model.layers.0.weight": "model.safetensors"},
+                    }
+                )
+            )
+        async with aiofiles.open(model_dir / "model.safetensors", "wb") as f:
+            await f.write(b"x" * 256)
+        async with aiofiles.open(model_dir / "model.safetensors.partial", "wb") as f:
+            await f.write(b"x" * 128)
+
+        assert is_model_directory_complete(model_dir) is True
+
+    async def test_returns_false_when_index_is_invalid(
+        self, model_id: ModelId, tmp_path: Path
+    ) -> None:
+        model_dir = tmp_path / model_id.normalize()
+        await aios.makedirs(model_dir, exist_ok=True)
+
+        async with aiofiles.open(model_dir / "model.safetensors.index.json", "w") as f:
+            await f.write("{not valid json")
+        async with aiofiles.open(model_dir / "model.safetensors", "wb") as f:
+            await f.write(b"x" * 256)
+
+        assert is_model_directory_complete(model_dir) is False
+
+
 class TestProgressResetOnRedownload:
     """Tests for progress tracking when files are re-downloaded."""
 

src/exo/download/tests/test_re_download.py

@@ -2,12 +2,16 @@
 
 import asyncio
 import contextlib
+import json
 from collections.abc import AsyncIterator, Awaitable
 from datetime import timedelta
 from pathlib import Path
 from typing import Callable
 from unittest.mock import AsyncMock, patch
 
+import aiofiles
+import aiofiles.os as aios
+
 from exo.download.coordinator import DownloadCoordinator
 from exo.download.download_utils import RepoDownloadProgress
 from exo.download.impl_shard_downloader import SingletonShardDownloader
@@ -192,6 +196,53 @@ async def test_re_download_after_delete_completes() -> None:
                 await coordinator_task
 
 
+async def test_start_download_uses_complete_local_model_without_status_probe(
+    tmp_path: Path,
+) -> None:
+    _, cmd_recv = channel[ForwarderDownloadCommand]()
+    event_send, event_recv = channel[Event]()
+
+    fake_downloader = FakeShardDownloader()
+    fake_downloader.get_shard_download_status_for_shard = AsyncMock(
+        side_effect=AssertionError("status probe should be skipped for complete models")
+    )
+
+    coordinator = DownloadCoordinator(
+        node_id=NODE_ID,
+        shard_downloader=SingletonShardDownloader(fake_downloader),
+        download_command_receiver=cmd_recv,
+        event_sender=event_send,
+    )
+    shard = _make_shard()
+
+    models_dir = tmp_path / "models"
+    model_dir = models_dir / MODEL_ID.normalize()
+    await aios.makedirs(model_dir, exist_ok=True)
+
+    async with aiofiles.open(model_dir / "config.json", "w") as f:
+        await f.write('{"model_type":"qwen2"}')
+    async with aiofiles.open(model_dir / "model.safetensors.index.json", "w") as f:
+        await f.write(
+                json.dumps(
+                    {
+                        "metadata": {"total_size": 128},
+                        "weight_map": {"model.layers.0.weight": "model.safetensors"},
+                    }
+                )
+            )
+    async with aiofiles.open(model_dir / "model.safetensors", "wb") as f:
+        await f.write(b"x" * 128)
+
+    with patch("exo.download.coordinator.EXO_MODELS_DIR", models_dir):
+        await coordinator._start_download(shard)  # pyright: ignore[reportPrivateUsage]
+
+    event = await event_recv.receive()
+    assert isinstance(event, NodeDownloadProgress)
+    assert isinstance(event.download_progress, DownloadCompleted)
+    assert event.download_progress.model_directory == str(model_dir)
+    assert event.download_progress.total == shard.model_card.storage_size
+
+
 async def _wait_for_download_completed(
     event_recv: Receiver[Event], model_id: ModelId, timeout: float = 2.0
 ) -> DownloadCompleted | None:

src/exo/main.py

@@ -11,9 +11,9 @@ from loguru import logger
 from pydantic import PositiveInt
 
 import exo.routing.topics as topics
+from exo.api.main import API
 from exo.download.coordinator import DownloadCoordinator
 from exo.download.impl_shard_downloader import exo_shard_downloader
-from exo.master.api import API  # TODO: should API be in master?
 from exo.master.main import Master
 from exo.routing.event_router import EventRouter
 from exo.routing.router import Router, get_node_id_keypair

src/exo/master/main.py

@@ -3,7 +3,6 @@ from datetime import datetime, timedelta, timezone
 import anyio
 from loguru import logger
 
-from exo.master.event_log import DiskEventLog
 from exo.master.placement import (
     add_instance_to_placements,
     cancel_unnecessary_downloads,
@@ -61,6 +60,7 @@ from exo.shared.types.tasks import (
 )
 from exo.shared.types.worker.instances import InstanceId
 from exo.utils.channels import Receiver, Sender
+from exo.utils.disk_event_log import DiskEventLog
 from exo.utils.event_buffer import MultiSourceBuffer
 from exo.utils.task_group import TaskGroup
 

src/exo/shared/apply.py

@@ -115,7 +115,13 @@ def apply_node_download_progress(event: NodeDownloadProgress, state: State) -> S
 
     replaced = False
     for i, existing_dp in enumerate(current):
-        if existing_dp.shard_metadata == dp.shard_metadata:
+        # TODO(ciaran): deduplicate by model_id for now. Will need to use
+        # shard_metadata again when pipeline and tensor downloads differ.
+        # For now this is fine
+        if (
+            existing_dp.shard_metadata.model_card.model_id
+            == dp.shard_metadata.model_card.model_id
+        ):
             current[i] = dp
             replaced = True
             break

src/exo/shared/types/chunks.py

@@ -1,18 +1,18 @@
 from collections.abc import Generator
 from typing import Any, Literal
 
-from exo.shared.models.model_cards import ModelId
-from exo.shared.types.api import (
+from exo.api.types import (
+    FinishReason,
     GenerationStats,
     ImageGenerationStats,
+    ToolCallItem,
     TopLogprobItem,
     Usage,
 )
+from exo.shared.models.model_cards import ModelId
 from exo.utils.pydantic_ext import TaggedModel
 
-from .api import FinishReason
 from .common import CommandId
-from .worker.runner_response import ToolCallItem
 
 
 class BaseChunk(TaggedModel):

src/exo/shared/types/commands.py

@@ -1,10 +1,10 @@
 from pydantic import Field
 
-from exo.shared.models.model_cards import ModelCard, ModelId
-from exo.shared.types.api import (
+from exo.api.types import (
     ImageEditsTaskParams,
     ImageGenerationTaskParams,
 )
+from exo.shared.models.model_cards import ModelCard, ModelId
 from exo.shared.types.chunks import InputImageChunk
 from exo.shared.types.common import CommandId, NodeId, SystemId
 from exo.shared.types.text_generation import TextGenerationTaskParams

src/exo/shared/types/tasks.py

@@ -2,7 +2,7 @@ from enum import Enum
 
 from pydantic import Field
 
-from exo.shared.types.api import (
+from exo.api.types import (
     ImageEditsTaskParams,
     ImageGenerationTaskParams,
 )

src/exo/shared/types/worker/runner_response.py

@@ -1,7 +1,7 @@
 from collections.abc import Generator
 from typing import Any, Literal
 
-from exo.shared.types.api import (
+from exo.api.types import (
     FinishReason,
     GenerationStats,
     ImageGenerationStats,

src/exo/utils/info_gatherer/info_gatherer.py

@@ -287,8 +287,8 @@ class ThunderboltBridgeInfo(TaggedModel):
                     service_name=tb_service_name,
                 )
             )
-        except Exception as e:
-            logger.warning(f"Failed to gather Thunderbolt Bridge info: {e}")
+        except Exception:
+            logger.opt(exception=True).warning("Failed to gather Thunderbolt Bridge info")
             return None
 
 
@@ -382,18 +382,69 @@ class InfoGatherer:
     rdma_ctl_poll_interval: float | None = 10 if IS_DARWIN else None
     disk_poll_interval: float | None = 30
     _tg: TaskGroup = field(init=False, default_factory=TaskGroup)
+    _psutil_memory_fallback_enabled: bool = field(init=False, default=False)
+
+    def _enable_psutil_memory_fallback(self, reason: str) -> None:
+        if not self._psutil_memory_fallback_enabled:
+            logger.warning(reason)
+            self._psutil_memory_fallback_enabled = True
+        self.memory_poll_rate = 1
+
+    def _get_macmon_path(self) -> str | None:
+        return os.getenv("EXO_MACMON_PATH") or shutil.which("macmon")
+
+    async def _can_read_macmon_metrics(self, macmon_path: str) -> bool:
+        try:
+            with fail_after(5):
+                proc = await anyio.run_process(
+                    [macmon_path, "pipe", "--samples", "1", "--interval", "100"],
+                    check=False,
+                )
+        except Exception:
+            logger.opt(exception=True).warning(
+                f"Failed to validate macmon at {macmon_path}"
+            )
+            return False
+
+        if proc.returncode != 0:
+            stderr = proc.stderr.decode("utf-8", errors="replace").strip()
+            logger.warning(
+                f"macmon preflight failed with return code {proc.returncode}: "
+                f"{stderr or 'no stderr'}"
+            )
+            return False
+
+        stdout = proc.stdout.decode("utf-8", errors="replace").strip()
+        if not stdout:
+            logger.warning("macmon preflight returned no metrics")
+            return False
+
+        try:
+            MacmonMetrics.from_raw_json(stdout.splitlines()[0])
+        except ValidationError:
+            logger.opt(exception=True).warning(
+                "macmon preflight returned unexpected metrics JSON"
+            )
+            return False
+
+        return True
 
     async def run(self):
         async with self._tg as tg:
             if IS_DARWIN:
-                if (macmon_path := shutil.which("macmon")) is not None:
-                    tg.start_soon(self._monitor_macmon, macmon_path)
+                if (macmon_path := self._get_macmon_path()) is not None:
+                    if await self._can_read_macmon_metrics(macmon_path):
+                        tg.start_soon(self._monitor_macmon, macmon_path)
+                    else:
+                        self._enable_psutil_memory_fallback(
+                            f"macmon at {macmon_path} is unusable, falling back "
+                            f"to psutil memory monitoring"
+                        )
                 else:
-                    # macmon not installed — fall back to psutil for memory
-                    logger.warning(
-                        "macmon not found, falling back to psutil for memory monitoring"
+                    self._enable_psutil_memory_fallback(
+                        "macmon not found, falling back to psutil for memory "
+                        "monitoring"
                     )
-                    self.memory_poll_rate = 1
                 tg.start_soon(self._monitor_system_profiler_thunderbolt_data)
                 tg.start_soon(self._monitor_thunderbolt_bridge_status)
                 tg.start_soon(self._monitor_rdma_ctl_status)
@@ -417,8 +468,8 @@ class InfoGatherer:
             try:
                 with fail_after(30):
                     await self.info_sender.send(await StaticNodeInformation.gather())
-            except Exception as e:
-                logger.warning(f"Error gathering static node info: {e}")
+            except Exception:
+                logger.opt(exception=True).warning("Error gathering static node info")
             await anyio.sleep(self.static_info_poll_interval)
 
     async def _monitor_misc(self):
@@ -428,8 +479,8 @@ class InfoGatherer:
             try:
                 with fail_after(10):
                     await self.info_sender.send(await MiscData.gather())
-            except Exception as e:
-                logger.warning(f"Error gathering misc data: {e}")
+            except Exception:
+                logger.opt(exception=True).warning("Error gathering misc data")
             await anyio.sleep(self.misc_poll_interval)
 
     async def _monitor_system_profiler_thunderbolt_data(self):
@@ -455,8 +506,8 @@ class InfoGatherer:
 
                     conns = [it for i in data if (it := i.conn()) is not None]
                     await self.info_sender.send(MacThunderboltConnections(conns=conns))
-            except Exception as e:
-                logger.warning(f"Error gathering Thunderbolt data: {e}")
+            except Exception:
+                logger.opt(exception=True).warning("Error gathering Thunderbolt data")
             await anyio.sleep(self.system_profiler_interval)
 
     async def _monitor_memory_usage(self):
@@ -466,16 +517,18 @@ class InfoGatherer:
             if override_memory_env
             else None
         )
-        if self.memory_poll_rate is None:
-            return
         while True:
+            poll_rate = self.memory_poll_rate
+            if poll_rate is None:
+                await anyio.sleep(1)
+                continue
             try:
                 await self.info_sender.send(
                     MemoryUsage.from_psutil(override_memory=override_memory)
                 )
-            except Exception as e:
-                logger.warning(f"Error gathering memory usage: {e}")
-            await anyio.sleep(self.memory_poll_rate)
+            except Exception:
+                logger.opt(exception=True).warning("Error gathering memory usage")
+            await anyio.sleep(poll_rate)
 
     async def _watch_system_info(self):
         if self.interface_watcher_interval is None:
@@ -485,8 +538,8 @@ class InfoGatherer:
                 with fail_after(10):
                     nics = await get_network_interfaces()
                     await self.info_sender.send(NodeNetworkInterfaces(ifaces=nics))
-            except Exception as e:
-                logger.warning(f"Error gathering network interfaces: {e}")
+            except Exception:
+                logger.opt(exception=True).warning("Error gathering network interfaces")
             await anyio.sleep(self.interface_watcher_interval)
 
     async def _monitor_thunderbolt_bridge_status(self):
@@ -498,8 +551,8 @@ class InfoGatherer:
                     curr = await ThunderboltBridgeInfo.gather()
                     if curr is not None:
                         await self.info_sender.send(curr)
-            except Exception as e:
-                logger.warning(f"Error gathering Thunderbolt Bridge status: {e}")
+            except Exception:
+                logger.opt(exception=True).warning("Error gathering Thunderbolt Bridge status")
             await anyio.sleep(self.thunderbolt_bridge_poll_interval)
 
     async def _monitor_rdma_ctl_status(self):
@@ -510,8 +563,8 @@ class InfoGatherer:
                 curr = await RdmaCtlStatus.gather()
                 if curr is not None:
                     await self.info_sender.send(curr)
-            except Exception as e:
-                logger.warning(f"Error gathering RDMA ctl status: {e}")
+            except Exception:
+                logger.opt(exception=True).warning("Error gathering RDMA ctl status")
             await anyio.sleep(self.rdma_ctl_poll_interval)
 
     async def _monitor_disk_usage(self):
@@ -521,8 +574,8 @@ class InfoGatherer:
             try:
                 with fail_after(5):
                     await self.info_sender.send(await NodeDiskUsage.gather())
-            except Exception as e:
-                logger.warning(f"Error gathering disk usage: {e}")
+            except Exception:
+                logger.opt(exception=True).warning("Error gathering disk usage")
             await anyio.sleep(self.disk_poll_interval)
 
     async def _monitor_macmon(self, macmon_path: str):
@@ -554,6 +607,10 @@ class InfoGatherer:
                 logger.warning(
                     f"MacMon produced no output for {read_timeout}s, restarting"
                 )
+                self._enable_psutil_memory_fallback(
+                    "MacMon produced no output, falling back to psutil memory "
+                    "monitoring"
+                )
             except CalledProcessError as e:
                 stderr_msg = "no stderr"
                 stderr_output = cast(bytes | str | None, e.stderr)
@@ -566,6 +623,12 @@ class InfoGatherer:
                 logger.warning(
                     f"MacMon failed with return code {e.returncode}: {stderr_msg}"
                 )
-            except Exception as e:
-                logger.warning(f"Error in macmon monitor: {e}")
+                self._enable_psutil_memory_fallback(
+                    "MacMon failed, falling back to psutil memory monitoring"
+                )
+            except Exception:
+                logger.opt(exception=True).warning("Error in macmon monitor")
+                self._enable_psutil_memory_fallback(
+                    "MacMon crashed, falling back to psutil memory monitoring"
+                )
             await anyio.sleep(self.macmon_interval)

src/exo/utils/power_sampler.py

@@ -5,7 +5,7 @@ from typing import final
 
 import anyio
 
-from exo.shared.types.api import NodePowerStats, PowerUsage
+from exo.api.types import NodePowerStats, PowerUsage
 from exo.shared.types.common import NodeId
 from exo.shared.types.profiling import SystemPerformanceProfile
 

src/exo/utils/tests/test_event_log.py

@@ -2,8 +2,8 @@ from pathlib import Path
 
 import pytest
 
-from exo.master.event_log import DiskEventLog
 from exo.shared.types.events import TestEvent
+from exo.utils.disk_event_log import DiskEventLog
 
 
 @pytest.fixture

src/exo/utils/tests/test_power_sampler.py

@@ -3,7 +3,7 @@ from collections.abc import Mapping
 import anyio
 import pytest
 
-from exo.shared.types.api import PowerUsage
+from exo.api.types import PowerUsage
 from exo.shared.types.common import NodeId
 from exo.shared.types.profiling import SystemPerformanceProfile
 from exo.utils.power_sampler import PowerSampler

src/exo/worker/engines/image/distributed_model.py

@@ -6,8 +6,8 @@ import mlx.core as mx
 from mflux.models.common.config.config import Config
 from PIL import Image
 
+from exo.api.types import AdvancedImageParams
 from exo.download.download_utils import build_model_path
-from exo.shared.types.api import AdvancedImageParams
 from exo.shared.types.worker.instances import BoundInstance
 from exo.shared.types.worker.shards import CfgShardMetadata, PipelineShardMetadata
 from exo.worker.engines.image.config import ImageModelConfig

src/exo/worker/engines/image/generate.py

@@ -9,7 +9,7 @@ from typing import Generator, Literal
 import mlx.core as mx
 from PIL import Image
 
-from exo.shared.types.api import (
+from exo.api.types import (
     AdvancedImageParams,
     ImageEditsTaskParams,
     ImageGenerationStats,

src/exo/worker/engines/mlx/dsml_encoding.py

@@ -4,7 +4,7 @@ from typing import Any
 
 from mlx_lm.chat_templates import deepseek_v32
 
-from exo.shared.types.api import ToolCallItem
+from exo.api.types import ToolCallItem
 
 BOS_TOKEN: str = deepseek_v32.bos_token
 EOS_TOKEN: str = deepseek_v32.eos_token

src/exo/worker/engines/mlx/generator/batch_generate.py

@@ -10,7 +10,7 @@ from mlx_lm.models.cache import RotatingKVCache
 from mlx_lm.sample_utils import make_logits_processors, make_sampler
 from mlx_lm.tokenizer_utils import StreamingDetokenizer, TokenizerWrapper
 
-from exo.shared.types.api import (
+from exo.api.types import (
     CompletionTokensDetails,
     FinishReason,
     GenerationStats,

src/exo/worker/engines/mlx/generator/generate.py

@@ -13,7 +13,7 @@ from mlx_lm.models.cache import ArraysCache, RotatingKVCache
 from mlx_lm.sample_utils import make_logits_processors, make_sampler
 from mlx_lm.tokenizer_utils import TokenizerWrapper
 
-from exo.shared.types.api import (
+from exo.api.types import (
     CompletionTokensDetails,
     FinishReason,
     GenerationStats,

src/exo/worker/main.py

@@ -5,10 +5,10 @@ import anyio
 from anyio import fail_after
 from loguru import logger
 
+from exo.api.types import ImageEditsTaskParams
 from exo.download.download_utils import resolve_model_in_path
 from exo.shared.apply import apply
 from exo.shared.models.model_cards import ModelId
-from exo.shared.types.api import ImageEditsTaskParams
 from exo.shared.types.commands import (
     ForwarderCommand,
     ForwarderDownloadCommand,

src/exo/worker/runner/image_models/runner.py

@@ -4,10 +4,10 @@ from typing import TYPE_CHECKING, Literal
 
 import mlx.core as mx
 
+from exo.api.types import ImageGenerationStats
 from exo.shared.constants import EXO_MAX_CHUNK_SIZE, EXO_TRACING_ENABLED
 from exo.shared.models.model_cards import ModelTask
 from exo.shared.tracing import clear_trace_buffer, get_trace_buffer
-from exo.shared.types.api import ImageGenerationStats
 from exo.shared.types.chunks import ErrorChunk, ImageChunk
 from exo.shared.types.common import CommandId, ModelId
 from exo.shared.types.events import (

src/exo/worker/runner/llm_inference/model_output_parsers.py

@@ -13,7 +13,7 @@ from openai_harmony import (  # pyright: ignore[reportMissingTypeStubs]
     load_harmony_encoding,
 )
 
-from exo.shared.types.api import ToolCallItem
+from exo.api.types import ToolCallItem
 from exo.shared.types.common import ModelId
 from exo.shared.types.mlx import Model
 from exo.shared.types.worker.runner_response import GenerationResponse, ToolCallResponse

src/exo/worker/runner/llm_inference/tool_parsers.py

@@ -3,7 +3,7 @@ import math
 from dataclasses import dataclass
 from typing import Any, Callable
 
-from exo.shared.types.api import ToolCallItem
+from exo.api.types import ToolCallItem
 
 
 @dataclass

src/exo/worker/tests/unittests/test_runner/test_parse_gpt_oss.py

@@ -1,6 +1,6 @@
 from collections.abc import Generator
 
-from exo.shared.types.api import FinishReason
+from exo.api.types import FinishReason
 from exo.shared.types.worker.runner_response import (
     GenerationResponse,
     ToolCallResponse,