fix: RDMA warning uses all edges to find peers, not just RDMA-tagged ones

Root cause: when RDMA is broken (ibv_alloc_pd failure), the
RDMAConnection edges are often absent from the topology — only
SocketConnection edges remain. The old code filtered to RDMA-tagged
edges only, so unhealthy nodes with no RDMA edges fell through to the
"on device X" solo fallback.

Fix: use ALL topology edges to find connected peers. Any direct
connection between nodes represents a Thunderbolt cable to re-seat.
Remove the solo fallback and the {#if pair.nodeB} template branch
entirely — warnings always say "between A and B".

Co-Authored-By: Claude Opus 4.6 <[email protected]>
This commit is contained in:
Alex Cheema
2026-02-17 10:23:11 -08:00
co-authored by Claude Opus 4.6
parent 59e44df71b
commit afaf778ba6
+9 -26
View File
@@ -205,32 +205,29 @@
.map(([id]) => id);
if (unhealthyNodeIds.length === 0) return null;
// Cross-reference with RDMA topology edges to identify cable endpoints
// Cross-reference with topology edges to identify cable endpoints.
// Use ALL edges (not just RDMA-tagged ones) because when RDMA is broken,
// the RDMAConnection edges may be absent — only SocketConnection edges remain.
// Any direct connection between nodes represents a Thunderbolt cable to re-seat.
const edges = data?.edges ?? [];
const seenPairs = new Set<string>();
const affectedPairs: Array<{
nodeA: string;
nodeB: string;
nameA: string;
nameB: string;
}> = [];
const seenPairs = new Set<string>();
for (const unhealthyId of unhealthyNodeIds) {
let foundPeer = false;
for (const edge of edges) {
if (!(edge.sourceRdmaIface || edge.sinkRdmaIface)) continue;
let peer: string | null = null;
if (edge.source === unhealthyId) peer = edge.target;
else if (edge.target === unhealthyId) peer = edge.source;
if (!peer) continue;
const pairKey = [unhealthyId, peer].sort().join("-");
if (seenPairs.has(pairKey)) {
foundPeer = true;
continue;
}
if (seenPairs.has(pairKey)) continue;
seenPairs.add(pairKey);
foundPeer = true;
affectedPairs.push({
nodeA: unhealthyId,
@@ -239,15 +236,6 @@
nameB: getNodeName(peer),
});
}
// If no RDMA edges found but node is unhealthy, still report it
if (!foundPeer) {
affectedPairs.push({
nodeA: unhealthyId,
nodeB: "",
nameA: getNodeName(unhealthyId),
nameB: "",
});
}
}
return affectedPairs.length > 0 ? affectedPairs : null;
@@ -2196,14 +2184,9 @@
{#each rdmaDeviceUnhealthy as pair}
<p class="text-xs text-white/60 mb-1">
<span class="text-red-300">To fix:</span>
{#if pair.nodeB}
Disconnect and reconnect the Thunderbolt 5 cable between
<strong class="text-white">{pair.nameA}</strong> and
<strong class="text-white">{pair.nameB}</strong>.
{:else}
Disconnect and reconnect the Thunderbolt 5 cable on
<strong class="text-white">{pair.nameA}</strong>.
{/if}
Disconnect and reconnect the Thunderbolt 5 cable between
<strong class="text-white">{pair.nameA}</strong> and
<strong class="text-white">{pair.nameB}</strong>.
</p>
{/each}
</div>