diff --git a/README.md b/README.md index 995edac5..58d41c37 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,10 @@ uv run exo This starts the exo dashboard and API at http://localhost:52415/ + +*Please view the section on RDMA to enable this feature on MacOS >=26.2!* + + ### Run from Source (Linux) **Prerequisites:** @@ -230,7 +234,7 @@ This removes: RDMA is a new capability added to macOS 26.2. It works on any Mac with Thunderbolt 5 (M4 Pro Mac Mini, M4 Max Mac Studio, M4 Max MacBook Pro, M3 Ultra Mac Studio). -Note that on Mac Studio, you cannot use the Thunderbolt 5 port next to the Ethernet port. +Please refer to the caveats for immediate troubleshooting. To enable RDMA on macOS, follow these steps: @@ -247,6 +251,14 @@ To enable RDMA on macOS, follow these steps: After that, RDMA will be enabled in macOS and exo will take care of the rest. +**Important Caveats** + +1. Devices that wish to be part of an RDMA cluster must be connected to all other devices in the cluster. +2. The cables must support TB5. +3. On a Mac Studio, you cannot use the Thunderbolt 5 port next to the Ethernet port. +4. If running from source, please use the script found at `tmp/set_rdma_network_config.sh`, which will disable Thunderbolt Bridge and set dhcp on each RDMA port. +5. RDMA ports may be unable to discover each other on different versions of MacOS. Please ensure that OS versions match exactly (even beta version numbers) on all devices. + --- ### Using the API diff --git a/tmp/set_rdma_network_config.sh b/tmp/set_rdma_network_config.sh new file mode 100755 index 00000000..e6a7f14d --- /dev/null +++ b/tmp/set_rdma_network_config.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -euo pipefail + +PREFS="/Library/Preferences/SystemConfiguration/preferences.plist" + +# Remove bridge0 interface +ifconfig bridge0 &>/dev/null && { + ifconfig bridge0 | grep -q 'member' && { + ifconfig bridge0 | awk '/member/ {print $2}' | xargs -n1 ifconfig bridge0 deletem 2>/dev/null || true + } + ifconfig bridge0 destroy 2>/dev/null || true +} + +# Remove Thunderbolt Bridge from VirtualNetworkInterfaces in preferences.plist +/usr/libexec/PlistBuddy -c "Delete :VirtualNetworkInterfaces:Bridge:bridge0" "$PREFS" 2>/dev/null || true + +networksetup -listlocations | grep -q exo || { + networksetup -createlocation exo +} + +networksetup -switchtolocation exo +networksetup -listallhardwareports \ + | awk -F': ' '/Hardware Port: / {print $2}' \ + | while IFS=":" read -r name; do + case "$name" in + "Ethernet Adapter"*) + ;; + "Thunderbolt Bridge") + ;; + "Thunderbolt "*) + networksetup -listallnetworkservices \ + | grep -q "EXO $name" \ + || networksetup -createnetworkservice "EXO $name" "$name" 2>/dev/null \ + || continue + networksetup -setdhcp "EXO $name" + ;; + *) + networksetup -listallnetworkservices \ + | grep -q "$name" \ + || networksetup -createnetworkservice "$name" "$name" 2>/dev/null \ + || continue + ;; + esac + done + +networksetup -listnetworkservices | grep -q "Thunderbolt Bridge" && { + networksetup -setnetworkserviceenabled "Thunderbolt Bridge" off +} || true