feat: add intermediate model-fit state for cluster-capacity-only models (#1441)

## Motivation
In the model picker, non-runnable models were not clearly separated
between two different cases:

- model exceeds currently available RAM but can fit in total cluster
capacity
- model exceeds total cluster capacity

That made it harder to distinguish "not runnable right now" from "too
large for this cluster."

## Changes
- Added a tri-state fit status in dashboard model-picker flow:
  - `fits_now`
  - `fits_cluster_capacity`
  - `too_large`
- Updated dashboard logic to compute both available cluster RAM and
total cluster RAM.
- Passed fit status through picker components.
- Updated model size color mapping:
  - `fits_now` -> white/gray
  - `fits_cluster_capacity` -> orange
  - `too_large` -> red
- Updated group ordering for non-runnable models:
- orange groups (`fits_cluster_capacity`) are listed above red groups
(`too_large`).

## Why It Works
Launch safety is unchanged: selection is still gated by existing
placement feasibility (`canModelFit`), so models that cannot run now
remain disabled.

The new fit status is used for visual distinction and ordering only:
- runnable now
- fits cluster capacity but not free RAM now
- too large for cluster capacity

## Before
Non-runnable models were not clearly distinguished by temporary capacity
vs hard capacity limit.

## After
Model picker clearly separates states by both color and order:
- Runnable now (white/gray)
- Fits cluster capacity but not free RAM now (orange, disabled)
- Exceeds cluster capacity (red, disabled)

## Test Plan
### Manual testing
- Open model picker with live cluster memory telemetry.
- Verify white/gray models are selectable.
- Verify orange models are disabled and appear above red models.
- Verify red models are disabled and appear below orange models.

### Automated checks
- `npm --prefix dashboard run build` passes.
- `uv run basedpyright` passes.
- `uv run ruff check` passes.
- `npm --prefix dashboard run check` reports existing pre-change Svelte
diagnostics (same known SVG title/a11y items).
- `uv run pytest` in this local environment exits during collection due
existing `tests/start_distributed_test.py` `SystemExit` usage.
- No Python code was changed.

---------

Co-authored-by: Alex Cheema <[email protected]>

dashboard/src/lib/components/ModelPickerGroup.svelte

@@ -26,6 +26,7 @@
     nodeNames: string[];
     nodeIds: string[];
   };
+  type ModelFitStatus = "fits_now" | "fits_cluster_capacity" | "too_large";
 
   type ModelPickerGroupProps = {
     group: ModelGroup;
@@ -33,6 +34,7 @@
     isFavorite: boolean;
     selectedModelId: string | null;
     canModelFit: (id: string) => boolean;
+    getModelFitStatus: (id: string) => ModelFitStatus;
     onToggleExpand: () => void;
     onSelectModel: (modelId: string) => void;
     onToggleFavorite: (baseModelId: string) => void;
@@ -46,6 +48,7 @@
     isFavorite,
     selectedModelId,
     canModelFit,
+    getModelFitStatus,
     onToggleExpand,
     onSelectModel,
     onToggleFavorite,
@@ -76,6 +79,30 @@
   const anyVariantFits = $derived(
     group.variants.some((v) => canModelFit(v.id)),
   );
+  const groupFitStatus = $derived.by((): ModelFitStatus => {
+    let hasClusterCapacityOnly = false;
+    for (const variant of group.variants) {
+      const fitStatus = getModelFitStatus(variant.id);
+      if (fitStatus === "fits_now") {
+        return "fits_now";
+      }
+      if (fitStatus === "fits_cluster_capacity") {
+        hasClusterCapacityOnly = true;
+      }
+    }
+    return hasClusterCapacityOnly ? "fits_cluster_capacity" : "too_large";
+  });
+
+  function getSizeClassForFitStatus(fitStatus: ModelFitStatus): string {
+    switch (fitStatus) {
+      case "fits_now":
+        return "text-white/40";
+      case "fits_cluster_capacity":
+        return "text-orange-400/80";
+      case "too_large":
+        return "text-red-400/70";
+    }
+  }
 
   // Check if this group's model is currently selected (for single-variant groups)
   const isMainSelected = $derived(
@@ -244,7 +271,14 @@
 
     <!-- Size indicator (smallest variant) -->
     {#if !group.hasMultipleVariants && group.smallestVariant?.storage_size_megabytes}
-      <span class="text-xs font-mono text-white/30 flex-shrink-0">
+      {@const singleVariantFitStatus = getModelFitStatus(
+        group.smallestVariant.id,
+      )}
+      <span
+        class="text-xs font-mono flex-shrink-0 {getSizeClassForFitStatus(
+          singleVariantFitStatus,
+        )}"
+      >
         {formatSize(group.smallestVariant.storage_size_megabytes)}
       </span>
     {/if}
@@ -255,7 +289,11 @@
         .map((v) => v.storage_size_megabytes || 0)
         .filter((s) => s > 0)
         .sort((a, b) => a - b)}
-      <span class="text-xs font-mono text-white/30 flex-shrink-0">
+      <span
+        class="text-xs font-mono flex-shrink-0 {getSizeClassForFitStatus(
+          groupFitStatus,
+        )}"
+      >
         {group.variants.length} variants{#if sizes.length >= 2}{" "}({formatSize(
             sizes[0],
           )}-{formatSize(sizes[sizes.length - 1])}){/if}
@@ -360,6 +398,7 @@
   {#if isExpanded && group.hasMultipleVariants}
     <div class="bg-black/20 border-t border-white/5">
       {#each group.variants as variant}
+        {@const fitStatus = getModelFitStatus(variant.id)}
         {@const modelCanFit = canModelFit(variant.id)}
         {@const isSelected = selectedModelId === variant.id}
         <button
@@ -384,7 +423,11 @@
           </span>
 
           <!-- Size -->
-          <span class="text-xs font-mono text-white/40 flex-1">
+          <span
+            class="text-xs font-mono flex-1 {getSizeClassForFitStatus(
+              fitStatus,
+            )}"
+          >
             {formatSize(variant.storage_size_megabytes)}
           </span>
 

dashboard/src/lib/components/ModelPickerModal.svelte

@@ -46,6 +46,8 @@
     tags: string[];
   }
 
+  type ModelFitStatus = "fits_now" | "fits_cluster_capacity" | "too_large";
+
   type ModelPickerModalProps = {
     isOpen: boolean;
     models: ModelInfo[];
@@ -53,6 +55,7 @@
     favorites: Set<string>;
     existingModelIds: Set<string>;
     canModelFit: (modelId: string) => boolean;
+    getModelFitStatus: (modelId: string) => ModelFitStatus;
     onSelect: (modelId: string) => void;
     onClose: () => void;
     onToggleFavorite: (baseModelId: string) => void;
@@ -78,6 +81,7 @@
     favorites,
     existingModelIds,
     canModelFit,
+    getModelFitStatus,
     onSelect,
     onClose,
     onToggleFavorite,
@@ -427,13 +431,23 @@
       );
     }
 
-    // Sort: models that fit first, then by size (largest first)
+    // Sort: fits-now first, then fits-cluster-capacity, then too-large
     result.sort((a, b) => {
-      const aFits = a.variants.some((v) => canModelFit(v.id));
-      const bFits = b.variants.some((v) => canModelFit(v.id));
+      const getGroupFitRank = (group: ModelGroup): number => {
+        let hasClusterCapacityOnly = false;
+        for (const variant of group.variants) {
+          const fitStatus = getModelFitStatus(variant.id);
+          if (fitStatus === "fits_now") return 0;
+          if (fitStatus === "fits_cluster_capacity") {
+            hasClusterCapacityOnly = true;
+          }
+        }
+        return hasClusterCapacityOnly ? 1 : 2;
+      };
 
-      if (aFits && !bFits) return -1;
-      if (!aFits && bFits) return 1;
+      const aRank = getGroupFitRank(a);
+      const bRank = getGroupFitRank(b);
+      if (aRank !== bRank) return aRank - bRank;
 
       return (
         (b.smallestVariant.storage_size_megabytes || 0) -
@@ -742,6 +756,7 @@
               isFavorite={favorites.has(group.id)}
               {selectedModelId}
               {canModelFit}
+              {getModelFitStatus}
               onToggleExpand={() => toggleGroupExpanded(group.id)}
               onSelectModel={handleSelect}
               {onToggleFavorite}

dashboard/src/routes/+page.svelte

@@ -113,6 +113,10 @@
       capabilities?: string[];
     }>
   >([]);
+  type ModelMemoryFitStatus =
+    | "fits_now"
+    | "fits_cluster_capacity"
+    | "too_large";
 
   // Model tasks lookup for ChatForm - maps both short IDs and full HuggingFace IDs
   const modelTasks = $derived(() => {
@@ -450,14 +454,41 @@
     );
   });
 
+  // Calculate total memory in the cluster (in GB)
+  const clusterTotalMemoryGB = $derived(() => {
+    if (!data) return 0;
+    return (
+      Object.values(data.nodes).reduce((acc, n) => {
+        const total =
+          n.macmon_info?.memory?.ram_total ?? n.system_info?.memory ?? 0;
+        return acc + total;
+      }, 0) /
+      (1024 * 1024 * 1024)
+    );
+  });
+
+  function getModelMemoryFitStatus(model: {
+    id: string;
+    name?: string;
+    storage_size_megabytes?: number;
+  }): ModelMemoryFitStatus {
+    const modelSizeGB = getModelSizeGB(model);
+    if (modelSizeGB <= availableMemoryGB()) {
+      return "fits_now";
+    }
+    if (modelSizeGB <= clusterTotalMemoryGB()) {
+      return "fits_cluster_capacity";
+    }
+    return "too_large";
+  }
+
   // Check if a model has enough memory to run
   function hasEnoughMemory(model: {
     id: string;
     name?: string;
     storage_size_megabytes?: number;
   }): boolean {
-    const modelSizeGB = getModelSizeGB(model);
-    return modelSizeGB <= availableMemoryGB();
+    return getModelMemoryFitStatus(model) === "fits_now";
   }
 
   // Sorted models for dropdown - biggest first, unrunnable at the end
@@ -3257,6 +3288,10 @@
     const model = models.find((m) => m.id === modelId);
     return model ? hasEnoughMemory(model) : false;
   }}
+  getModelFitStatus={(modelId): ModelMemoryFitStatus => {
+    const model = models.find((m) => m.id === modelId);
+    return model ? getModelMemoryFitStatus(model) : "too_large";
+  }}
   onSelect={handleModelPickerSelect}
   onClose={() => (isModelPickerOpen = false)}
   onToggleFavorite={toggleFavorite}