fix: shard redistribution blocked when pinned node dies (BUG-B)

When a MetaInstance was created with node_ids (from the placement
preview), try_place_for_meta_instance passed them as required_nodes
to place_instance, which demands ALL listed nodes exist in a topology
cycle. If a pinned node died and was removed from the topology, no
cycle could satisfy the constraint and placement failed permanently.

Fix: intersect node_ids with live topology nodes before passing as
required_nodes. Dead nodes are dropped from the constraint so
placement succeeds on the remaining N-1 nodes.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

src/exo/master/reconcile.py

@@ -212,15 +212,24 @@ def try_place_for_meta_instance(
         min_nodes=meta_instance.min_nodes,
     )
     try:
+        # Intersect requested node_ids with live topology nodes so that dead
+        # nodes don't block re-placement.  If the intersection is empty (all
+        # requested nodes are gone), drop the constraint entirely and let the
+        # placement algorithm pick the best available nodes.
+        live_nodes = set(topology.list_nodes())
+        required_nodes: set[NodeId] | None = None
+        if meta_instance.node_ids:
+            alive = set(meta_instance.node_ids) & live_nodes
+            if alive:
+                required_nodes = alive
+
         target_instances = place_instance(
             command,
             topology,
             current_instances,
             node_memory,
             node_network,
-            required_nodes=(
-                set(meta_instance.node_ids) if meta_instance.node_ids else None
-            ),
+            required_nodes=required_nodes,
         )
         # Tag the new instance with meta_instance_id
         new_instance_ids = set(target_instances.keys()) - set(current_instances.keys())

src/exo/master/tests/test_reconcile.py

@@ -4,6 +4,7 @@ from exo.master.reconcile import (
     instance_connections_healthy,
     instance_runners_failed,
     instance_satisfies_meta_instance,
+    try_place_for_meta_instance,
 )
 from exo.shared.apply import apply
 from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
@@ -740,3 +741,52 @@ async def test_health_reconciler_network_failure_always_deletes():
     assert len(events) == 1
     assert isinstance(events[0], InstanceDeleted)
     assert events[0].failure_error == "Network connection lost"
+
+
+# --- try_place_for_meta_instance: dead node_ids recovery (BUG-B) ---
+
+
+async def test_replaces_instance_when_pinned_node_dies():
+    """When a MetaInstance has node_ids pinned to [A,B,C,D] and node D dies,
+    re-placement should succeed on the remaining [A,B,C] instead of failing
+    because the dead node is no longer in the topology."""
+    from exo.shared.types.profiling import MemoryUsage
+
+    alive_nodes = ["node-a", "node-b", "node-c"]
+    dead_node = "node-d"
+    all_nodes = alive_nodes + [dead_node]
+
+    # MetaInstance was originally pinned to all 4 nodes
+    meta = _meta_instance(
+        node_ids=[NodeId(n) for n in all_nodes],
+        min_nodes=1,
+    )
+
+    # Topology has only the 3 alive nodes (dead node already timed out)
+    topology = _topology(*alive_nodes)
+
+    # Enough memory on each alive node
+    node_memory = {
+        NodeId(n): MemoryUsage(
+            ram_total=Memory.from_gb(32),
+            ram_available=Memory.from_gb(16),
+            swap_total=Memory.from_gb(0),
+            swap_available=Memory.from_gb(0),
+        )
+        for n in alive_nodes
+    }
+
+    model_card = _model_card()
+    result = try_place_for_meta_instance(
+        meta,
+        model_card,
+        topology,
+        current_instances={},
+        node_memory=node_memory,
+        node_network={},
+    )
+
+    # Placement should succeed — not blocked by the dead node
+    assert result.error is None
+    assert len(result.events) > 0
+    assert isinstance(result.events[0], InstanceCreated)