From dccf2440ba2f453720d6069396fa0fae1858e92c Mon Sep 17 00:00:00 2001 From: Alex Cheema Date: Wed, 11 Feb 2026 10:01:07 -0800 Subject: [PATCH] Add placement error feedback and per-node loading status Show why MetaInstance placement fails instead of stuck "PLACING", and show per-node runner status during loading for multi-node instances. Co-Authored-By: Claude Opus 4.6 --- dashboard/src/lib/stores/app.svelte.ts | 5 + dashboard/src/routes/+page.svelte | 311 +++++++++++++----- src/exo/master/main.py | 17 +- .../master/process_managers/meta_instance.py | 21 +- src/exo/master/reconcile.py | 21 +- src/exo/shared/apply.py | 35 +- src/exo/shared/types/events.py | 7 + src/exo/shared/types/state.py | 1 + 8 files changed, 318 insertions(+), 100 deletions(-) diff --git a/dashboard/src/lib/stores/app.svelte.ts b/dashboard/src/lib/stores/app.svelte.ts index 25ec59b6..421ca01e 100644 --- a/dashboard/src/lib/stores/app.svelte.ts +++ b/dashboard/src/lib/stores/app.svelte.ts @@ -233,6 +233,8 @@ interface RawStateResponse { thunderboltBridgeCycles?: string[][]; // MetaInstances (declarative instance constraints) metaInstances?: Record; + // MetaInstance placement errors + metaInstanceErrors?: Record; } export interface MetaInstanceData { @@ -506,6 +508,7 @@ class AppStore { previewNodeFilter = $state>(new Set()); lastUpdate = $state(null); metaInstances = $state>({}); + metaInstanceErrors = $state>({}); thunderboltBridgeCycles = $state([]); nodeThunderboltBridge = $state< Record< @@ -1215,6 +1218,7 @@ class AppStore { } // MetaInstances this.metaInstances = data.metaInstances ?? {}; + this.metaInstanceErrors = data.metaInstanceErrors ?? {}; // Thunderbolt bridge cycles this.thunderboltBridgeCycles = data.thunderboltBridgeCycles ?? []; // Thunderbolt bridge status per node @@ -2966,6 +2970,7 @@ export const totalTokens = () => appStore.totalTokens; export const topologyData = () => appStore.topologyData; export const instances = () => appStore.instances; export const metaInstances = () => appStore.metaInstances; +export const metaInstanceErrors = () => appStore.metaInstanceErrors; export const runners = () => appStore.runners; export const downloads = () => appStore.downloads; export const placementPreviews = () => appStore.placementPreviews; diff --git a/dashboard/src/routes/+page.svelte b/dashboard/src/routes/+page.svelte index 61fa6782..64ec195d 100644 --- a/dashboard/src/routes/+page.svelte +++ b/dashboard/src/routes/+page.svelte @@ -38,6 +38,7 @@ chatSidebarVisible, toggleChatSidebarVisible, metaInstances, + metaInstanceErrors, thunderboltBridgeCycles, nodeThunderboltBridge, type DownloadProgress, @@ -63,18 +64,43 @@ const topologyOnlyEnabled = $derived(topologyOnlyMode()); const sidebarVisible = $derived(chatSidebarVisible()); const metaInstancesData = $derived(metaInstances()); + const metaInstanceErrorsData = $derived(metaInstanceErrors()); const tbBridgeCycles = $derived(thunderboltBridgeCycles()); - // Shared fallback objects for MetaInstances without a backing instance yet - const PLACING_STATUS = { - statusText: "PLACING", - statusClass: "starting", - isDownloading: false, - isFailed: false, - progress: null, - perNode: [], - errorMessage: null, - } as const; + // Get status for a MetaInstance that has no backing instance yet + function getMetaInstancePlacingStatus(metaInstanceId: string) { + const error = metaInstanceErrorsData[metaInstanceId]; + if (error) { + return { + statusText: "PLACEMENT FAILED", + statusClass: "failed", + isDownloading: false as const, + isFailed: true, + progress: null, + perNode: [] as Array<{ + nodeId: string; + nodeName: string; + progress: DownloadProgress; + }>, + perNodeStatus: [] as PerNodeRunnerStatus[], + errorMessage: error, + }; + } + return { + statusText: "PLACING", + statusClass: "starting", + isDownloading: false as const, + isFailed: false, + progress: null, + perNode: [] as Array<{ + nodeId: string; + nodeName: string; + progress: DownloadProgress; + }>, + perNodeStatus: [] as PerNodeRunnerStatus[], + errorMessage: null, + }; + } const tbBridgeData = $derived(nodeThunderboltBridge()); const nodeFilter = $derived(previewNodeFilter()); @@ -845,15 +871,18 @@ nodeName: string; progress: DownloadProgress; }>; + perNodeStatus: PerNodeRunnerStatus[]; } { if (!downloadsData || Object.keys(downloadsData).length === 0) { + const statusInfo = deriveInstanceStatus(instanceWrapped); return { isDownloading: false, isFailed: false, errorMessage: null, progress: null, - statusText: "RUNNING", + statusText: statusInfo.statusText, perNode: [], + perNodeStatus: statusInfo.perNodeStatus, }; } @@ -867,6 +896,7 @@ progress: null, statusText: "PREPARING", perNode: [], + perNodeStatus: [], }; } @@ -935,6 +965,7 @@ progress: null, statusText: "FAILED", perNode: [], + perNodeStatus: [], }; } } @@ -979,6 +1010,7 @@ progress: null, statusText: statusInfo.statusText, perNode: [], + perNodeStatus: statusInfo.perNodeStatus, }; } @@ -1002,92 +1034,161 @@ }, statusText: "DOWNLOADING", perNode, + perNodeStatus: [], }; } // Derive instance status from runners // Get color class for a status function getStatusColor(statusText: string): string { - switch (statusText) { - case "FAILED": - return "text-red-400"; - case "SHUTDOWN": - return "text-gray-400"; - case "DOWNLOADING": - return "text-blue-400"; - case "LOADING": - case "WARMING UP": - case "WAITING": - case "INITIALIZING": - return "text-yellow-400"; - case "RUNNING": - return "text-teal-400"; - case "READY": - case "LOADED": - return "text-green-400"; - default: - return "text-exo-light-gray"; - } + if (statusText === "FAILED" || statusText === "PLACEMENT FAILED") + return "text-red-400"; + if (statusText === "SHUTDOWN") return "text-gray-400"; + if (statusText === "DOWNLOADING") return "text-blue-400"; + if ( + statusText.startsWith("LOADING") || + statusText.startsWith("WARMING UP") || + statusText === "WAITING" || + statusText === "INITIALIZING" + ) + return "text-yellow-400"; + if (statusText === "RUNNING") return "text-teal-400"; + if (statusText === "READY" || statusText === "LOADED") + return "text-green-400"; + return "text-exo-light-gray"; + } + + const RUNNER_STATUS_MAP: Record = { + RunnerWaitingForInitialization: "WaitingForInitialization", + RunnerInitializingBackend: "InitializingBackend", + RunnerWaitingForModel: "WaitingForModel", + RunnerLoading: "Loading", + RunnerLoaded: "Loaded", + RunnerWarmingUp: "WarmingUp", + RunnerReady: "Ready", + RunnerRunning: "Running", + RunnerShutdown: "Shutdown", + RunnerFailed: "Failed", + }; + + // Friendly labels for display + const RUNNER_STATUS_DISPLAY: Record = { + WaitingForInitialization: "Initializing", + InitializingBackend: "Initializing", + WaitingForModel: "Waiting", + Loading: "Loading", + Loaded: "Loaded", + WarmingUp: "Warming Up", + Ready: "Ready", + Running: "Running", + Shutdown: "Shutdown", + Failed: "Failed", + }; + + interface PerNodeRunnerStatus { + nodeId: string; + nodeName: string; + status: string; // friendly display status } function deriveInstanceStatus(instanceWrapped: unknown): { statusText: string; statusClass: string; + perNodeStatus: PerNodeRunnerStatus[]; } { const [, instance] = getTagged(instanceWrapped); if (!instance || typeof instance !== "object") { - return { statusText: "PREPARING", statusClass: "inactive" }; + return { + statusText: "PREPARING", + statusClass: "inactive", + perNodeStatus: [], + }; } const inst = instance as { - shardAssignments?: { runnerToShard?: Record }; + shardAssignments?: { + runnerToShard?: Record; + nodeToRunner?: Record; + }; }; + const nodeToRunner = inst.shardAssignments?.nodeToRunner || {}; const runnerIds = Object.keys(inst.shardAssignments?.runnerToShard || {}); + const totalNodes = runnerIds.length; - const statuses = runnerIds - .map((rid) => { - const r = runnersData[rid]; - if (!r) return null; + // Build per-node status + const perNodeStatus: PerNodeRunnerStatus[] = []; + const statuses: string[] = []; + for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) { + const r = runnersData[runnerId]; + let status: string | null = null; + if (r) { const [kind] = getTagged(r); - const statusMap: Record = { - RunnerWaitingForInitialization: "WaitingForInitialization", - RunnerInitializingBackend: "InitializingBackend", - RunnerWaitingForModel: "WaitingForModel", - RunnerLoading: "Loading", - RunnerLoaded: "Loaded", - RunnerWarmingUp: "WarmingUp", - RunnerReady: "Ready", - RunnerRunning: "Running", - RunnerShutdown: "Shutdown", - RunnerFailed: "Failed", - }; - return kind ? statusMap[kind] || null : null; - }) - .filter((s): s is string => s !== null); + status = kind ? RUNNER_STATUS_MAP[kind] || null : null; + } + if (status) { + statuses.push(status); + perNodeStatus.push({ + nodeId, + nodeName: getNodeName(nodeId), + status: RUNNER_STATUS_DISPLAY[status] || status, + }); + } + } const has = (s: string) => statuses.includes(s); + const count = (s: string) => statuses.filter((v) => v === s).length; if (statuses.length === 0) - return { statusText: "PREPARING", statusClass: "inactive" }; - if (has("Failed")) return { statusText: "FAILED", statusClass: "failed" }; + return { + statusText: "PREPARING", + statusClass: "inactive", + perNodeStatus, + }; + if (has("Failed")) + return { statusText: "FAILED", statusClass: "failed", perNodeStatus }; if (has("Shutdown")) - return { statusText: "SHUTDOWN", statusClass: "inactive" }; - if (has("Loading")) - return { statusText: "LOADING", statusClass: "starting" }; - if (has("WarmingUp")) - return { statusText: "WARMING UP", statusClass: "starting" }; - if (has("Running")) - return { statusText: "RUNNING", statusClass: "running" }; - if (has("Ready")) return { statusText: "READY", statusClass: "loaded" }; - if (has("Loaded")) return { statusText: "LOADED", statusClass: "loaded" }; - if (has("WaitingForModel")) - return { statusText: "WAITING", statusClass: "starting" }; - if (has("InitializingBackend")) - return { statusText: "INITIALIZING", statusClass: "starting" }; - if (has("WaitingForInitialization")) - return { statusText: "INITIALIZING", statusClass: "starting" }; + return { statusText: "SHUTDOWN", statusClass: "inactive", perNodeStatus }; - return { statusText: "RUNNING", statusClass: "active" }; + // For loading/warming states, show node progress when multi-node + if (has("Loading")) { + const readyCount = count("Ready") + count("Running") + count("Loaded"); + const statusText = + totalNodes > 1 + ? `LOADING (${readyCount}/${totalNodes} nodes ready)` + : "LOADING"; + return { statusText, statusClass: "starting", perNodeStatus }; + } + if (has("WarmingUp")) { + const readyCount = count("Ready") + count("Running"); + const statusText = + totalNodes > 1 + ? `WARMING UP (${readyCount}/${totalNodes} nodes ready)` + : "WARMING UP"; + return { statusText, statusClass: "starting", perNodeStatus }; + } + + if (has("Running")) + return { statusText: "RUNNING", statusClass: "running", perNodeStatus }; + if (has("Ready")) + return { statusText: "READY", statusClass: "loaded", perNodeStatus }; + if (has("Loaded")) + return { statusText: "LOADED", statusClass: "loaded", perNodeStatus }; + if (has("WaitingForModel")) + return { statusText: "WAITING", statusClass: "starting", perNodeStatus }; + if (has("InitializingBackend")) + return { + statusText: "INITIALIZING", + statusClass: "starting", + perNodeStatus, + }; + if (has("WaitingForInitialization")) + return { + statusText: "INITIALIZING", + statusClass: "starting", + perNodeStatus, + }; + + return { statusText: "RUNNING", statusClass: "active", perNodeStatus }; } function getBytes(value: unknown): number { @@ -2097,13 +2198,15 @@ {@const instance = item.instance} {@const downloadInfo = instance ? getInstanceDownloadStatus(item.instanceId ?? id, instance) - : PLACING_STATUS} + : getMetaInstancePlacingStatus(id)} {@const statusText = downloadInfo.statusText} {@const isDownloading = downloadInfo.isDownloading} - {@const isFailed = statusText === "FAILED"} + {@const isFailed = + statusText === "FAILED" || + statusText === "PLACEMENT FAILED"} {@const isLoading = - statusText === "LOADING" || - statusText === "WARMING UP" || + statusText.startsWith("LOADING") || + statusText.startsWith("WARMING UP") || statusText === "WAITING" || statusText === "PLACING"} {@const isReady = @@ -2114,7 +2217,12 @@ {@const instanceInfo = instance ? getInstanceInfo(instance) : { - instanceType: item.instanceMeta === "MlxRing" ? "MLX Ring" : item.instanceMeta === "MlxJaccl" ? "MLX RDMA" : "Unknown", + instanceType: + item.instanceMeta === "MlxRing" + ? "MLX Ring" + : item.instanceMeta === "MlxJaccl" + ? "MLX RDMA" + : "Unknown", sharding: item.sharding ?? "Unknown", nodeNames: [] as string[], nodeIds: [] as string[], @@ -2509,6 +2617,24 @@ {downloadInfo.errorMessage} {/if} + {#if downloadInfo.perNodeStatus.length > 1 && (statusText.startsWith("LOADING") || statusText.startsWith("WARMING UP") || statusText === "WAITING" || statusText === "INITIALIZING")} +
+ {#each downloadInfo.perNodeStatus as node} +
+ {node.nodeName} + {node.status} +
+ {/each} +
+ {/if} {/if} @@ -2950,13 +3076,15 @@ item.instanceId ?? id, instance, ) - : PLACING_STATUS} + : getMetaInstancePlacingStatus(id)} {@const statusText = downloadInfo.statusText} {@const isDownloading = downloadInfo.isDownloading} - {@const isFailed = statusText === "FAILED"} + {@const isFailed = + statusText === "FAILED" || + statusText === "PLACEMENT FAILED"} {@const isLoading = - statusText === "LOADING" || - statusText === "WARMING UP" || + statusText.startsWith("LOADING") || + statusText.startsWith("WARMING UP") || statusText === "WAITING" || statusText === "PLACING"} {@const isReady = @@ -2967,7 +3095,12 @@ {@const instanceInfo = instance ? getInstanceInfo(instance) : { - instanceType: item.instanceMeta === "MlxRing" ? "MLX Ring" : item.instanceMeta === "MlxJaccl" ? "MLX RDMA" : "Unknown", + instanceType: + item.instanceMeta === "MlxRing" + ? "MLX Ring" + : item.instanceMeta === "MlxJaccl" + ? "MLX RDMA" + : "Unknown", sharding: item.sharding ?? "Unknown", nodeNames: [] as string[], nodeIds: [] as string[], @@ -3372,6 +3505,24 @@ {downloadInfo.errorMessage} {/if} + {#if downloadInfo.perNodeStatus.length > 1 && (statusText.startsWith("LOADING") || statusText.startsWith("WARMING UP") || statusText === "WAITING" || statusText === "INITIALIZING")} +
+ {#each downloadInfo.perNodeStatus as node} +
+ {node.nodeName} + {node.status} +
+ {/each} +
+ {/if} {/if} diff --git a/src/exo/master/main.py b/src/exo/master/main.py index 6efa843f..bdfeac14 100644 --- a/src/exo/master/main.py +++ b/src/exo/master/main.py @@ -303,16 +303,15 @@ class Master: model_card = await ModelCard.load( command.meta_instance.model_id ) - generated_events.extend( - try_place_for_meta_instance( - command.meta_instance, - model_card, - self.state.topology, - self.state.instances, - self.state.node_memory, - self.state.node_network, - ) + result = try_place_for_meta_instance( + command.meta_instance, + model_card, + self.state.topology, + self.state.instances, + self.state.node_memory, + self.state.node_network, ) + generated_events.extend(result.events) case DeleteMetaInstance(): generated_events.append( MetaInstanceDeleted( diff --git a/src/exo/master/process_managers/meta_instance.py b/src/exo/master/process_managers/meta_instance.py index d94de6bb..f4825b0d 100644 --- a/src/exo/master/process_managers/meta_instance.py +++ b/src/exo/master/process_managers/meta_instance.py @@ -6,7 +6,7 @@ from exo.master.reconcile import ( try_place_for_meta_instance, ) from exo.shared.models.model_cards import ModelCard -from exo.shared.types.events import Event, InstanceCreated +from exo.shared.types.events import Event, InstanceCreated, MetaInstancePlacementFailed from exo.shared.types.state import State from exo.shared.types.worker.instances import Instance, InstanceId @@ -28,7 +28,7 @@ class MetaInstanceReconciler: ) for meta_instance in unsatisfied: model_card = await ModelCard.load(meta_instance.model_id) - events = try_place_for_meta_instance( + result = try_place_for_meta_instance( meta_instance, model_card, state.topology, @@ -37,8 +37,21 @@ class MetaInstanceReconciler: state.node_network, ) # Update local instance map so next placement sees this one - for event in events: + for event in result.events: if isinstance(event, InstanceCreated): current_instances[event.instance.instance_id] = event.instance - all_events.extend(events) + all_events.extend(result.events) + + # Emit placement failure if error differs from what's already in state + if result.error is not None: + existing_error = state.meta_instance_errors.get( + meta_instance.meta_instance_id + ) + if existing_error != result.error: + all_events.append( + MetaInstancePlacementFailed( + meta_instance_id=meta_instance.meta_instance_id, + reason=result.error, + ) + ) return all_events diff --git a/src/exo/master/reconcile.py b/src/exo/master/reconcile.py index 094df056..86318362 100644 --- a/src/exo/master/reconcile.py +++ b/src/exo/master/reconcile.py @@ -1,4 +1,5 @@ from collections.abc import Mapping, Sequence +from typing import NamedTuple from loguru import logger @@ -20,6 +21,13 @@ from exo.shared.types.worker.instances import ( ) +class PlacementResult(NamedTuple): + """Result of a placement attempt: events to apply and optional error reason.""" + + events: Sequence[Event] + error: str | None + + def _get_ring_order(instance: BaseInstance) -> list[NodeId]: """Reconstruct ring order from shard device_rank.""" node_ranks: list[tuple[NodeId, int]] = [] @@ -130,11 +138,11 @@ def try_place_for_meta_instance( current_instances: Mapping[InstanceId, Instance], node_memory: Mapping[NodeId, MemoryUsage], node_network: Mapping[NodeId, NodeNetworkInfo], -) -> Sequence[Event]: +) -> PlacementResult: """Try to place an instance satisfying the meta-instance constraints. - Returns InstanceCreated events on success, empty sequence on failure. - The new instance carries ``meta_instance_id`` so the binding is implicit. + Returns a :class:`PlacementResult` with events on success, or an error + reason on failure. """ command = PlaceInstance( model_card=model_card, @@ -160,9 +168,12 @@ def try_place_for_meta_instance( target_instances[new_id] = target_instances[new_id].model_copy( update={"meta_instance_id": meta_instance.meta_instance_id} ) - return list(get_transition_events(current_instances, target_instances)) + return PlacementResult( + events=list(get_transition_events(current_instances, target_instances)), + error=None, + ) except ValueError as e: logger.debug( f"MetaInstance placement not possible for {meta_instance.model_id}: {e}" ) - return [] + return PlacementResult(events=[], error=str(e)) diff --git a/src/exo/shared/apply.py b/src/exo/shared/apply.py index e73658c2..55ca1bac 100644 --- a/src/exo/shared/apply.py +++ b/src/exo/shared/apply.py @@ -14,6 +14,7 @@ from exo.shared.types.events import ( InstanceDeleted, MetaInstanceCreated, MetaInstanceDeleted, + MetaInstancePlacementFailed, NodeDownloadProgress, NodeGatheredInfo, NodeTimedOut, @@ -76,6 +77,8 @@ def event_apply(event: Event, state: State) -> State: return apply_meta_instance_created(event, state) case MetaInstanceDeleted(): return apply_meta_instance_deleted(event, state) + case MetaInstancePlacementFailed(): + return apply_meta_instance_placement_failed(event, state) case NodeTimedOut(): return apply_node_timed_out(event, state) case NodeDownloadProgress(): @@ -184,7 +187,18 @@ def apply_instance_created(event: InstanceCreated, state: State) -> State: **state.instances, instance.instance_id: instance, } - return state.model_copy(update={"instances": new_instances}) + update: dict[str, object] = {"instances": new_instances} + # Clear placement error when an instance is created for a meta-instance + if ( + instance.meta_instance_id + and instance.meta_instance_id in state.meta_instance_errors + ): + update["meta_instance_errors"] = { + mid: err + for mid, err in state.meta_instance_errors.items() + if mid != instance.meta_instance_id + } + return state.model_copy(update=update) def apply_instance_deleted(event: InstanceDeleted, state: State) -> State: @@ -208,7 +222,24 @@ def apply_meta_instance_deleted(event: MetaInstanceDeleted, state: State) -> Sta for mid, mi in state.meta_instances.items() if mid != event.meta_instance_id } - return state.model_copy(update={"meta_instances": new_meta}) + new_errors: Mapping[MetaInstanceId, str] = { + mid: err + for mid, err in state.meta_instance_errors.items() + if mid != event.meta_instance_id + } + return state.model_copy( + update={"meta_instances": new_meta, "meta_instance_errors": new_errors} + ) + + +def apply_meta_instance_placement_failed( + event: MetaInstancePlacementFailed, state: State +) -> State: + new_errors: Mapping[MetaInstanceId, str] = { + **state.meta_instance_errors, + event.meta_instance_id: event.reason, + } + return state.model_copy(update={"meta_instance_errors": new_errors}) def apply_runner_status_updated(event: RunnerStatusUpdated, state: State) -> State: diff --git a/src/exo/shared/types/events.py b/src/exo/shared/types/events.py index 370bc909..9339c911 100644 --- a/src/exo/shared/types/events.py +++ b/src/exo/shared/types/events.py @@ -77,6 +77,12 @@ class MetaInstanceDeleted(BaseEvent): meta_instance_id: MetaInstanceId +@final +class MetaInstancePlacementFailed(BaseEvent): + meta_instance_id: MetaInstanceId + reason: str + + class RunnerStatusUpdated(BaseEvent): runner_id: RunnerId runner_status: RunnerStatus @@ -152,6 +158,7 @@ Event = ( | InstanceDeleted | MetaInstanceCreated | MetaInstanceDeleted + | MetaInstancePlacementFailed | RunnerStatusUpdated | RunnerDeleted | NodeTimedOut diff --git a/src/exo/shared/types/state.py b/src/exo/shared/types/state.py index 054f8d75..9601c220 100644 --- a/src/exo/shared/types/state.py +++ b/src/exo/shared/types/state.py @@ -41,6 +41,7 @@ class State(CamelCaseModel): ) instances: Mapping[InstanceId, Instance] = {} meta_instances: Mapping[MetaInstanceId, MetaInstance] = {} + meta_instance_errors: Mapping[MetaInstanceId, str] = {} runners: Mapping[RunnerId, RunnerStatus] = {} downloads: Mapping[NodeId, Sequence[DownloadProgress]] = {} tasks: Mapping[TaskId, Task] = {}