Add placement error feedback and per-node loading status

Show why MetaInstance placement fails instead of stuck "PLACING", and show per-node runner status during loading for multi-node instances. Co-Authored-By: Claude Opus 4.6 <[email protected]>
2026-02-11 10:01:07 -08:00
parent f96f3f2c0f
commit dccf2440ba
8 changed files with 318 additions and 100 deletions
@@ -233,6 +233,8 @@ interface RawStateResponse {
  thunderboltBridgeCycles?: string[][];
  // MetaInstances (declarative instance constraints)
  metaInstances?: Record<string, MetaInstanceData>;
+  // MetaInstance placement errors
+  metaInstanceErrors?: Record<string, string>;
 }

 export interface MetaInstanceData {
@@ -506,6 +508,7 @@ class AppStore {
  previewNodeFilter = $state<Set<string>>(new Set());
  lastUpdate = $state<number | null>(null);
  metaInstances = $state<Record<string, MetaInstanceData>>({});
+  metaInstanceErrors = $state<Record<string, string>>({});
  thunderboltBridgeCycles = $state<string[][]>([]);
  nodeThunderboltBridge = $state<
    Record<
@@ -1215,6 +1218,7 @@ class AppStore {
      }
      // MetaInstances
      this.metaInstances = data.metaInstances ?? {};
+      this.metaInstanceErrors = data.metaInstanceErrors ?? {};
      // Thunderbolt bridge cycles
      this.thunderboltBridgeCycles = data.thunderboltBridgeCycles ?? [];
      // Thunderbolt bridge status per node
@@ -2966,6 +2970,7 @@ export const totalTokens = () => appStore.totalTokens;
 export const topologyData = () => appStore.topologyData;
 export const instances = () => appStore.instances;
 export const metaInstances = () => appStore.metaInstances;
+export const metaInstanceErrors = () => appStore.metaInstanceErrors;
 export const runners = () => appStore.runners;
 export const downloads = () => appStore.downloads;
 export const placementPreviews = () => appStore.placementPreviews;
@@ -38,6 +38,7 @@
    chatSidebarVisible,
    toggleChatSidebarVisible,
    metaInstances,
+    metaInstanceErrors,
    thunderboltBridgeCycles,
    nodeThunderboltBridge,
    type DownloadProgress,
@@ -63,18 +64,43 @@
  const topologyOnlyEnabled = $derived(topologyOnlyMode());
  const sidebarVisible = $derived(chatSidebarVisible());
  const metaInstancesData = $derived(metaInstances());
+  const metaInstanceErrorsData = $derived(metaInstanceErrors());
  const tbBridgeCycles = $derived(thunderboltBridgeCycles());

-  // Shared fallback objects for MetaInstances without a backing instance yet
-  const PLACING_STATUS = {
-    statusText: "PLACING",
-    statusClass: "starting",
-    isDownloading: false,
-    isFailed: false,
-    progress: null,
-    perNode: [],
-    errorMessage: null,
-  } as const;
+  // Get status for a MetaInstance that has no backing instance yet
+  function getMetaInstancePlacingStatus(metaInstanceId: string) {
+    const error = metaInstanceErrorsData[metaInstanceId];
+    if (error) {
+      return {
+        statusText: "PLACEMENT FAILED",
+        statusClass: "failed",
+        isDownloading: false as const,
+        isFailed: true,
+        progress: null,
+        perNode: [] as Array<{
+          nodeId: string;
+          nodeName: string;
+          progress: DownloadProgress;
+        }>,
+        perNodeStatus: [] as PerNodeRunnerStatus[],
+        errorMessage: error,
+      };
+    }
+    return {
+      statusText: "PLACING",
+      statusClass: "starting",
+      isDownloading: false as const,
+      isFailed: false,
+      progress: null,
+      perNode: [] as Array<{
+        nodeId: string;
+        nodeName: string;
+        progress: DownloadProgress;
+      }>,
+      perNodeStatus: [] as PerNodeRunnerStatus[],
+      errorMessage: null,
+    };
+  }

  const tbBridgeData = $derived(nodeThunderboltBridge());
  const nodeFilter = $derived(previewNodeFilter());
@@ -845,15 +871,18 @@
      nodeName: string;
      progress: DownloadProgress;
    }>;
+    perNodeStatus: PerNodeRunnerStatus[];
  } {
    if (!downloadsData || Object.keys(downloadsData).length === 0) {
+      const statusInfo = deriveInstanceStatus(instanceWrapped);
      return {
        isDownloading: false,
        isFailed: false,
        errorMessage: null,
        progress: null,
-        statusText: "RUNNING",
+        statusText: statusInfo.statusText,
        perNode: [],
+        perNodeStatus: statusInfo.perNodeStatus,
      };
    }

@@ -867,6 +896,7 @@
        progress: null,
        statusText: "PREPARING",
        perNode: [],
+        perNodeStatus: [],
      };
    }

@@ -935,6 +965,7 @@
              progress: null,
              statusText: "FAILED",
              perNode: [],
+              perNodeStatus: [],
            };
          }
        }
@@ -979,6 +1010,7 @@
        progress: null,
        statusText: statusInfo.statusText,
        perNode: [],
+        perNodeStatus: statusInfo.perNodeStatus,
      };
    }

@@ -1002,92 +1034,161 @@
      },
      statusText: "DOWNLOADING",
      perNode,
+      perNodeStatus: [],
    };
  }

  // Derive instance status from runners
  // Get color class for a status
  function getStatusColor(statusText: string): string {
-    switch (statusText) {
-      case "FAILED":
-        return "text-red-400";
-      case "SHUTDOWN":
-        return "text-gray-400";
-      case "DOWNLOADING":
-        return "text-blue-400";
-      case "LOADING":
-      case "WARMING UP":
-      case "WAITING":
-      case "INITIALIZING":
-        return "text-yellow-400";
-      case "RUNNING":
-        return "text-teal-400";
-      case "READY":
-      case "LOADED":
-        return "text-green-400";
-      default:
-        return "text-exo-light-gray";
-    }
+    if (statusText === "FAILED" || statusText === "PLACEMENT FAILED")
+      return "text-red-400";
+    if (statusText === "SHUTDOWN") return "text-gray-400";
+    if (statusText === "DOWNLOADING") return "text-blue-400";
+    if (
+      statusText.startsWith("LOADING") ||
+      statusText.startsWith("WARMING UP") ||
+      statusText === "WAITING" ||
+      statusText === "INITIALIZING"
+    )
+      return "text-yellow-400";
+    if (statusText === "RUNNING") return "text-teal-400";
+    if (statusText === "READY" || statusText === "LOADED")
+      return "text-green-400";
+    return "text-exo-light-gray";
+  }
+
+  const RUNNER_STATUS_MAP: Record<string, string> = {
+    RunnerWaitingForInitialization: "WaitingForInitialization",
+    RunnerInitializingBackend: "InitializingBackend",
+    RunnerWaitingForModel: "WaitingForModel",
+    RunnerLoading: "Loading",
+    RunnerLoaded: "Loaded",
+    RunnerWarmingUp: "WarmingUp",
+    RunnerReady: "Ready",
+    RunnerRunning: "Running",
+    RunnerShutdown: "Shutdown",
+    RunnerFailed: "Failed",
+  };
+
+  // Friendly labels for display
+  const RUNNER_STATUS_DISPLAY: Record<string, string> = {
+    WaitingForInitialization: "Initializing",
+    InitializingBackend: "Initializing",
+    WaitingForModel: "Waiting",
+    Loading: "Loading",
+    Loaded: "Loaded",
+    WarmingUp: "Warming Up",
+    Ready: "Ready",
+    Running: "Running",
+    Shutdown: "Shutdown",
+    Failed: "Failed",
+  };
+
+  interface PerNodeRunnerStatus {
+    nodeId: string;
+    nodeName: string;
+    status: string; // friendly display status
  }

  function deriveInstanceStatus(instanceWrapped: unknown): {
    statusText: string;
    statusClass: string;
+    perNodeStatus: PerNodeRunnerStatus[];
  } {
    const [, instance] = getTagged(instanceWrapped);
    if (!instance || typeof instance !== "object") {
-      return { statusText: "PREPARING", statusClass: "inactive" };
+      return {
+        statusText: "PREPARING",
+        statusClass: "inactive",
+        perNodeStatus: [],
+      };
    }

    const inst = instance as {
-      shardAssignments?: { runnerToShard?: Record<string, unknown> };
+      shardAssignments?: {
+        runnerToShard?: Record<string, unknown>;
+        nodeToRunner?: Record<string, string>;
+      };
    };
+    const nodeToRunner = inst.shardAssignments?.nodeToRunner || {};
    const runnerIds = Object.keys(inst.shardAssignments?.runnerToShard || {});
+    const totalNodes = runnerIds.length;

-    const statuses = runnerIds
-      .map((rid) => {
-        const r = runnersData[rid];
-        if (!r) return null;
+    // Build per-node status
+    const perNodeStatus: PerNodeRunnerStatus[] = [];
+    const statuses: string[] = [];
+    for (const [nodeId, runnerId] of Object.entries(nodeToRunner)) {
+      const r = runnersData[runnerId];
+      let status: string | null = null;
+      if (r) {
        const [kind] = getTagged(r);
-        const statusMap: Record<string, string> = {
-          RunnerWaitingForInitialization: "WaitingForInitialization",
-          RunnerInitializingBackend: "InitializingBackend",
-          RunnerWaitingForModel: "WaitingForModel",
-          RunnerLoading: "Loading",
-          RunnerLoaded: "Loaded",
-          RunnerWarmingUp: "WarmingUp",
-          RunnerReady: "Ready",
-          RunnerRunning: "Running",
-          RunnerShutdown: "Shutdown",
-          RunnerFailed: "Failed",
-        };
-        return kind ? statusMap[kind] || null : null;
-      })
-      .filter((s): s is string => s !== null);
+        status = kind ? RUNNER_STATUS_MAP[kind] || null : null;
+      }
+      if (status) {
+        statuses.push(status);
+        perNodeStatus.push({
+          nodeId,
+          nodeName: getNodeName(nodeId),
+          status: RUNNER_STATUS_DISPLAY[status] || status,
+        });
+      }
+    }

    const has = (s: string) => statuses.includes(s);
+    const count = (s: string) => statuses.filter((v) => v === s).length;

    if (statuses.length === 0)
-      return { statusText: "PREPARING", statusClass: "inactive" };
-    if (has("Failed")) return { statusText: "FAILED", statusClass: "failed" };
+      return {
+        statusText: "PREPARING",
+        statusClass: "inactive",
+        perNodeStatus,
+      };
+    if (has("Failed"))
+      return { statusText: "FAILED", statusClass: "failed", perNodeStatus };
    if (has("Shutdown"))
-      return { statusText: "SHUTDOWN", statusClass: "inactive" };
-    if (has("Loading"))
-      return { statusText: "LOADING", statusClass: "starting" };
-    if (has("WarmingUp"))
-      return { statusText: "WARMING UP", statusClass: "starting" };
-    if (has("Running"))
-      return { statusText: "RUNNING", statusClass: "running" };
-    if (has("Ready")) return { statusText: "READY", statusClass: "loaded" };
-    if (has("Loaded")) return { statusText: "LOADED", statusClass: "loaded" };
-    if (has("WaitingForModel"))
-      return { statusText: "WAITING", statusClass: "starting" };
-    if (has("InitializingBackend"))
-      return { statusText: "INITIALIZING", statusClass: "starting" };
-    if (has("WaitingForInitialization"))
-      return { statusText: "INITIALIZING", statusClass: "starting" };
+      return { statusText: "SHUTDOWN", statusClass: "inactive", perNodeStatus };

-    return { statusText: "RUNNING", statusClass: "active" };
+    // For loading/warming states, show node progress when multi-node
+    if (has("Loading")) {
+      const readyCount = count("Ready") + count("Running") + count("Loaded");
+      const statusText =
+        totalNodes > 1
+          ? `LOADING (${readyCount}/${totalNodes} nodes ready)`
+          : "LOADING";
+      return { statusText, statusClass: "starting", perNodeStatus };
+    }
+    if (has("WarmingUp")) {
+      const readyCount = count("Ready") + count("Running");
+      const statusText =
+        totalNodes > 1
+          ? `WARMING UP (${readyCount}/${totalNodes} nodes ready)`
+          : "WARMING UP";
+      return { statusText, statusClass: "starting", perNodeStatus };
+    }
+
+    if (has("Running"))
+      return { statusText: "RUNNING", statusClass: "running", perNodeStatus };
+    if (has("Ready"))
+      return { statusText: "READY", statusClass: "loaded", perNodeStatus };
+    if (has("Loaded"))
+      return { statusText: "LOADED", statusClass: "loaded", perNodeStatus };
+    if (has("WaitingForModel"))
+      return { statusText: "WAITING", statusClass: "starting", perNodeStatus };
+    if (has("InitializingBackend"))
+      return {
+        statusText: "INITIALIZING",
+        statusClass: "starting",
+        perNodeStatus,
+      };
+    if (has("WaitingForInitialization"))
+      return {
+        statusText: "INITIALIZING",
+        statusClass: "starting",
+        perNodeStatus,
+      };
+
+    return { statusText: "RUNNING", statusClass: "active", perNodeStatus };
  }

  function getBytes(value: unknown): number {
@@ -2097,13 +2198,15 @@
                  {@const instance = item.instance}
                  {@const downloadInfo = instance
                    ? getInstanceDownloadStatus(item.instanceId ?? id, instance)
-                    : PLACING_STATUS}
+                    : getMetaInstancePlacingStatus(id)}
                  {@const statusText = downloadInfo.statusText}
                  {@const isDownloading = downloadInfo.isDownloading}
-                  {@const isFailed = statusText === "FAILED"}
+                  {@const isFailed =
+                    statusText === "FAILED" ||
+                    statusText === "PLACEMENT FAILED"}
                  {@const isLoading =
-                    statusText === "LOADING" ||
-                    statusText === "WARMING UP" ||
+                    statusText.startsWith("LOADING") ||
+                    statusText.startsWith("WARMING UP") ||
                    statusText === "WAITING" ||
                    statusText === "PLACING"}
                  {@const isReady =
@@ -2114,7 +2217,12 @@
                  {@const instanceInfo = instance
                    ? getInstanceInfo(instance)
                    : {
-                        instanceType: item.instanceMeta === "MlxRing" ? "MLX Ring" : item.instanceMeta === "MlxJaccl" ? "MLX RDMA" : "Unknown",
+                        instanceType:
+                          item.instanceMeta === "MlxRing"
+                            ? "MLX Ring"
+                            : item.instanceMeta === "MlxJaccl"
+                              ? "MLX RDMA"
+                              : "Unknown",
                        sharding: item.sharding ?? "Unknown",
                        nodeNames: [] as string[],
                        nodeIds: [] as string[],
@@ -2509,6 +2617,24 @@
                              {downloadInfo.errorMessage}
                            </div>
                          {/if}
+                          {#if downloadInfo.perNodeStatus.length > 1 && (statusText.startsWith("LOADING") || statusText.startsWith("WARMING UP") || statusText === "WAITING" || statusText === "INITIALIZING")}
+                            <div class="mt-1.5 space-y-0.5">
+                              {#each downloadInfo.perNodeStatus as node}
+                                <div
+                                  class="flex items-center justify-between text-[10px] font-mono"
+                                >
+                                  <span class="text-white/60 truncate pr-2"
+                                    >{node.nodeName}</span
+                                  >
+                                  <span
+                                    class={getStatusColor(
+                                      node.status.toUpperCase(),
+                                    )}>{node.status}</span
+                                  >
+                                </div>
+                              {/each}
+                            </div>
+                          {/if}
                        {/if}
                      </div>
                    </div>
@@ -2950,13 +3076,15 @@
                          item.instanceId ?? id,
                          instance,
                        )
-                      : PLACING_STATUS}
+                      : getMetaInstancePlacingStatus(id)}
                    {@const statusText = downloadInfo.statusText}
                    {@const isDownloading = downloadInfo.isDownloading}
-                    {@const isFailed = statusText === "FAILED"}
+                    {@const isFailed =
+                      statusText === "FAILED" ||
+                      statusText === "PLACEMENT FAILED"}
                    {@const isLoading =
-                      statusText === "LOADING" ||
-                      statusText === "WARMING UP" ||
+                      statusText.startsWith("LOADING") ||
+                      statusText.startsWith("WARMING UP") ||
                      statusText === "WAITING" ||
                      statusText === "PLACING"}
                    {@const isReady =
@@ -2967,7 +3095,12 @@
                    {@const instanceInfo = instance
                      ? getInstanceInfo(instance)
                      : {
-                          instanceType: item.instanceMeta === "MlxRing" ? "MLX Ring" : item.instanceMeta === "MlxJaccl" ? "MLX RDMA" : "Unknown",
+                          instanceType:
+                            item.instanceMeta === "MlxRing"
+                              ? "MLX Ring"
+                              : item.instanceMeta === "MlxJaccl"
+                                ? "MLX RDMA"
+                                : "Unknown",
                          sharding: item.sharding ?? "Unknown",
                          nodeNames: [] as string[],
                          nodeIds: [] as string[],
@@ -3372,6 +3505,24 @@
                                {downloadInfo.errorMessage}
                              </div>
                            {/if}
+                            {#if downloadInfo.perNodeStatus.length > 1 && (statusText.startsWith("LOADING") || statusText.startsWith("WARMING UP") || statusText === "WAITING" || statusText === "INITIALIZING")}
+                              <div class="mt-1.5 space-y-0.5">
+                                {#each downloadInfo.perNodeStatus as node}
+                                  <div
+                                    class="flex items-center justify-between text-[10px] font-mono"
+                                  >
+                                    <span class="text-white/60 truncate pr-2"
+                                      >{node.nodeName}</span
+                                    >
+                                    <span
+                                      class={getStatusColor(
+                                        node.status.toUpperCase(),
+                                      )}>{node.status}</span
+                                    >
+                                  </div>
+                                {/each}
+                              </div>
+                            {/if}
                          {/if}
                        </div>
                      </div>
@@ -303,16 +303,15 @@ class Master:
                            model_card = await ModelCard.load(
                                command.meta_instance.model_id
                            )
-                            generated_events.extend(
-                                try_place_for_meta_instance(
-                                    command.meta_instance,
-                                    model_card,
-                                    self.state.topology,
-                                    self.state.instances,
-                                    self.state.node_memory,
-                                    self.state.node_network,
-                                )
+                            result = try_place_for_meta_instance(
+                                command.meta_instance,
+                                model_card,
+                                self.state.topology,
+                                self.state.instances,
+                                self.state.node_memory,
+                                self.state.node_network,
                            )
+                            generated_events.extend(result.events)
                        case DeleteMetaInstance():
                            generated_events.append(
                                MetaInstanceDeleted(
@@ -6,7 +6,7 @@ from exo.master.reconcile import (
    try_place_for_meta_instance,
 )
 from exo.shared.models.model_cards import ModelCard
-from exo.shared.types.events import Event, InstanceCreated
+from exo.shared.types.events import Event, InstanceCreated, MetaInstancePlacementFailed
 from exo.shared.types.state import State
 from exo.shared.types.worker.instances import Instance, InstanceId

@@ -28,7 +28,7 @@ class MetaInstanceReconciler:
        )
        for meta_instance in unsatisfied:
            model_card = await ModelCard.load(meta_instance.model_id)
-            events = try_place_for_meta_instance(
+            result = try_place_for_meta_instance(
                meta_instance,
                model_card,
                state.topology,
@@ -37,8 +37,21 @@ class MetaInstanceReconciler:
                state.node_network,
            )
            # Update local instance map so next placement sees this one
-            for event in events:
+            for event in result.events:
                if isinstance(event, InstanceCreated):
                    current_instances[event.instance.instance_id] = event.instance
-            all_events.extend(events)
+            all_events.extend(result.events)
+
+            # Emit placement failure if error differs from what's already in state
+            if result.error is not None:
+                existing_error = state.meta_instance_errors.get(
+                    meta_instance.meta_instance_id
+                )
+                if existing_error != result.error:
+                    all_events.append(
+                        MetaInstancePlacementFailed(
+                            meta_instance_id=meta_instance.meta_instance_id,
+                            reason=result.error,
+                        )
+                    )
        return all_events
@@ -1,4 +1,5 @@
 from collections.abc import Mapping, Sequence
+from typing import NamedTuple

 from loguru import logger

@@ -20,6 +21,13 @@ from exo.shared.types.worker.instances import (
 )


+class PlacementResult(NamedTuple):
+    """Result of a placement attempt: events to apply and optional error reason."""
+
+    events: Sequence[Event]
+    error: str | None
+
+
 def _get_ring_order(instance: BaseInstance) -> list[NodeId]:
    """Reconstruct ring order from shard device_rank."""
    node_ranks: list[tuple[NodeId, int]] = []
@@ -130,11 +138,11 @@ def try_place_for_meta_instance(
    current_instances: Mapping[InstanceId, Instance],
    node_memory: Mapping[NodeId, MemoryUsage],
    node_network: Mapping[NodeId, NodeNetworkInfo],
-) -> Sequence[Event]:
+) -> PlacementResult:
    """Try to place an instance satisfying the meta-instance constraints.

-    Returns InstanceCreated events on success, empty sequence on failure.
-    The new instance carries ``meta_instance_id`` so the binding is implicit.
+    Returns a :class:`PlacementResult` with events on success, or an error
+    reason on failure.
    """
    command = PlaceInstance(
        model_card=model_card,
@@ -160,9 +168,12 @@ def try_place_for_meta_instance(
            target_instances[new_id] = target_instances[new_id].model_copy(
                update={"meta_instance_id": meta_instance.meta_instance_id}
            )
-        return list(get_transition_events(current_instances, target_instances))
+        return PlacementResult(
+            events=list(get_transition_events(current_instances, target_instances)),
+            error=None,
+        )
    except ValueError as e:
        logger.debug(
            f"MetaInstance placement not possible for {meta_instance.model_id}: {e}"
        )
-        return []
+        return PlacementResult(events=[], error=str(e))
@@ -14,6 +14,7 @@ from exo.shared.types.events import (
    InstanceDeleted,
    MetaInstanceCreated,
    MetaInstanceDeleted,
+    MetaInstancePlacementFailed,
    NodeDownloadProgress,
    NodeGatheredInfo,
    NodeTimedOut,
@@ -76,6 +77,8 @@ def event_apply(event: Event, state: State) -> State:
            return apply_meta_instance_created(event, state)
        case MetaInstanceDeleted():
            return apply_meta_instance_deleted(event, state)
+        case MetaInstancePlacementFailed():
+            return apply_meta_instance_placement_failed(event, state)
        case NodeTimedOut():
            return apply_node_timed_out(event, state)
        case NodeDownloadProgress():
@@ -184,7 +187,18 @@ def apply_instance_created(event: InstanceCreated, state: State) -> State:
        **state.instances,
        instance.instance_id: instance,
    }
-    return state.model_copy(update={"instances": new_instances})
+    update: dict[str, object] = {"instances": new_instances}
+    # Clear placement error when an instance is created for a meta-instance
+    if (
+        instance.meta_instance_id
+        and instance.meta_instance_id in state.meta_instance_errors
+    ):
+        update["meta_instance_errors"] = {
+            mid: err
+            for mid, err in state.meta_instance_errors.items()
+            if mid != instance.meta_instance_id
+        }
+    return state.model_copy(update=update)


 def apply_instance_deleted(event: InstanceDeleted, state: State) -> State:
@@ -208,7 +222,24 @@ def apply_meta_instance_deleted(event: MetaInstanceDeleted, state: State) -> Sta
        for mid, mi in state.meta_instances.items()
        if mid != event.meta_instance_id
    }
-    return state.model_copy(update={"meta_instances": new_meta})
+    new_errors: Mapping[MetaInstanceId, str] = {
+        mid: err
+        for mid, err in state.meta_instance_errors.items()
+        if mid != event.meta_instance_id
+    }
+    return state.model_copy(
+        update={"meta_instances": new_meta, "meta_instance_errors": new_errors}
+    )
+
+
+def apply_meta_instance_placement_failed(
+    event: MetaInstancePlacementFailed, state: State
+) -> State:
+    new_errors: Mapping[MetaInstanceId, str] = {
+        **state.meta_instance_errors,
+        event.meta_instance_id: event.reason,
+    }
+    return state.model_copy(update={"meta_instance_errors": new_errors})


 def apply_runner_status_updated(event: RunnerStatusUpdated, state: State) -> State:
@@ -77,6 +77,12 @@ class MetaInstanceDeleted(BaseEvent):
    meta_instance_id: MetaInstanceId


+@final
+class MetaInstancePlacementFailed(BaseEvent):
+    meta_instance_id: MetaInstanceId
+    reason: str
+
+
 class RunnerStatusUpdated(BaseEvent):
    runner_id: RunnerId
    runner_status: RunnerStatus
@@ -152,6 +158,7 @@ Event = (
    | InstanceDeleted
    | MetaInstanceCreated
    | MetaInstanceDeleted
+    | MetaInstancePlacementFailed
    | RunnerStatusUpdated
    | RunnerDeleted
    | NodeTimedOut
@@ -41,6 +41,7 @@ class State(CamelCaseModel):
    )
    instances: Mapping[InstanceId, Instance] = {}
    meta_instances: Mapping[MetaInstanceId, MetaInstance] = {}
+    meta_instance_errors: Mapping[MetaInstanceId, str] = {}
    runners: Mapping[RunnerId, RunnerStatus] = {}
    downloads: Mapping[NodeId, Sequence[DownloadProgress]] = {}
    tasks: Mapping[TaskId, Task] = {}