or maybe this

we were seeing ooms loading minimax m2.5-8bit (230gib) on a 256gib m3
ultra - bumping required mem by 10% will prevent placements that
potentially exceed max memory

src/exo/master/placement.py

@@ -69,7 +69,17 @@ def place_instance(
     required_nodes: set[NodeId] | None = None,
 ) -> dict[InstanceId, Instance]:
     cycles = topology.get_cycles()
-    candidate_cycles = list(filter(lambda it: len(it) >= command.min_nodes, cycles))
+    candidate_cycles = filter(lambda it: len(it) >= command.min_nodes, cycles)
+    nodes_with_exo = set[NodeId]()
+    for instance in current_instances.values():
+        nodes_with_exo |= instance.shard_assignments.node_to_runner.keys()
+    candidate_cycles = [
+        Cycle(
+            cycle.node_ids,
+            has_other_placement=any(node in nodes_with_exo for node in cycle),
+        )
+        for cycle in candidate_cycles
+    ]
 
     # Filter to cycles containing all required nodes (subset matching)
     if required_nodes:
@@ -79,7 +89,7 @@ def place_instance(
             if required_nodes.issubset(cycle.node_ids)
         ]
     cycles_with_sufficient_memory = filter_cycles_by_memory(
-        candidate_cycles, node_memory, command.model_card.storage_size
+        candidate_cycles, node_memory, command.model_card.storage_size * 1.1
     )
     if len(cycles_with_sufficient_memory) == 0:
         raise ValueError("No cycles found with sufficient memory")

src/exo/master/placement_utils.py

@@ -29,7 +29,12 @@ def filter_cycles_by_memory(
             continue
 
         total_mem = sum(
-            (node_memory[node_id].ram_available for node_id in cycle.node_ids),
+            (
+                node_memory[node_id].ram_available
+                if cycle.has_other_placement
+                else node_memory[node_id].ram_total
+                for node_id in cycle.node_ids
+            ),
             start=Memory(),
         )
         if total_mem >= required_memory:

src/exo/shared/types/topology.py

@@ -9,6 +9,7 @@ from exo.utils.pydantic_ext import FrozenModel
 @dataclass(frozen=True)
 class Cycle:
     node_ids: list[NodeId]
+    has_other_placement: bool = False
 
     def __len__(self) -> int:
         return self.node_ids.__len__()