Merge branch 'main' into leo/dgx-spark-integrations

dashboard/src/routes/+page.svelte

@@ -1533,7 +1533,11 @@
           downloadKind
         ] as Record<string, unknown>;
 
-        if (downloadKind !== "DownloadOngoing") continue;
+        if (
+          downloadKind !== "DownloadOngoing" &&
+          downloadKind !== "DownloadPending"
+        )
+          continue;
         if (!downloadPayload) continue;
 
         const downloadModelId = extractModelIdFromDownload(downloadPayload);
@@ -1548,9 +1552,38 @@
           if (downloadModelId !== modelId) continue;
         }
 
-        isDownloading = true;
+        // For DownloadPending with partial bytes (paused/resumed downloads),
+        // synthesize a progress object from the top-level downloaded/total fields
+        let progress: DownloadProgress | null;
+        if (downloadKind === "DownloadPending") {
+          const pendingDownloaded = getBytes(
+            downloadPayload.downloaded ??
+              downloadPayload.downloaded_bytes ??
+              downloadPayload.downloadedBytes,
+          );
+          const pendingTotal = getBytes(
+            downloadPayload.total ??
+              downloadPayload.total_bytes ??
+              downloadPayload.totalBytes,
+          );
+          if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
+          isDownloading = true;
+          progress = {
+            totalBytes: pendingTotal,
+            downloadedBytes: pendingDownloaded,
+            speed: 0,
+            etaMs: 0,
+            percentage:
+              pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
+            completedFiles: 0,
+            totalFiles: 0,
+            files: [],
+          };
+        } else {
+          isDownloading = true;
+          progress = parseDownloadProgress(downloadPayload);
+        }
 
-        const progress = parseDownloadProgress(downloadPayload);
         if (progress) {
           // Sum all values across nodes - each node downloads independently
           totalBytes += progress.totalBytes;
@@ -1702,7 +1735,11 @@
           }
         }
 
-        if (downloadKind !== "DownloadOngoing") continue;
+        if (
+          downloadKind !== "DownloadOngoing" &&
+          downloadKind !== "DownloadPending"
+        )
+          continue;
         if (!downloadPayload) continue;
 
         // Check if this download is for this instance's model
@@ -1712,9 +1749,37 @@
           downloadModelId &&
           downloadModelId === instanceModelId
         ) {
-          isDownloading = true;
+          // For DownloadPending with partial bytes, synthesize progress
+          let progress: DownloadProgress | null;
+          if (downloadKind === "DownloadPending") {
+            const pendingDownloaded = getBytes(
+              downloadPayload.downloaded ??
+                downloadPayload.downloaded_bytes ??
+                downloadPayload.downloadedBytes,
+            );
+            const pendingTotal = getBytes(
+              downloadPayload.total ??
+                downloadPayload.total_bytes ??
+                downloadPayload.totalBytes,
+            );
+            if (pendingDownloaded <= 0 && pendingTotal <= 0) continue;
+            isDownloading = true;
+            progress = {
+              totalBytes: pendingTotal,
+              downloadedBytes: pendingDownloaded,
+              speed: 0,
+              etaMs: 0,
+              percentage:
+                pendingTotal > 0 ? (pendingDownloaded / pendingTotal) * 100 : 0,
+              completedFiles: 0,
+              totalFiles: 0,
+              files: [],
+            };
+          } else {
+            isDownloading = true;
+            progress = parseDownloadProgress(downloadPayload);
+          }
 
-          const progress = parseDownloadProgress(downloadPayload);
           if (progress) {
             // Sum all values across nodes - each node downloads independently
             totalBytes += progress.totalBytes;

src/exo/worker/engines/mlx/generator/batch_generate.py

@@ -291,10 +291,13 @@ class ExoBatchGenerator:
                     if generation_elapsed > 0
                     else 0.0
                 )
-                mlx_stats = self._exo_gen.stats()
+                try:
+                    mlx_stats = self._exo_gen.stats()
+                except ZeroDivisionError:
+                    mlx_stats = None
                 stats = GenerationStats(
                     prompt_tps=float(mlx_stats.prompt_tps)
-                    if mlx_stats.prompt_time > 0
+                    if mlx_stats is not None and mlx_stats.prompt_time > 0
                     else 0.0,
                     generation_tps=float(generation_tps),
                     prompt_tokens=len(state.all_prompt_tokens),