override macmon in flake (#1747)
updates macmon to an upstream fork that fixes m5 max issues. might see if the upstream version gets merged before we release. --------- Co-authored-by: Alex Cheema <alexcheema123@gmail.com>
This commit is contained in:
+11
-2
@@ -11,9 +11,18 @@ To run EXO from source:
|
||||
```bash
|
||||
brew install uv
|
||||
```
|
||||
- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
|
||||
- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
|
||||
```bash
|
||||
brew install macmon
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
rustup toolchain install nightly
|
||||
```
|
||||
- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
|
||||
Use the pinned fork revision used by this repo instead of Homebrew `macmon`.
|
||||
```bash
|
||||
cargo install --git https://github.com/swiftraccoon/macmon \
|
||||
--rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
|
||||
macmon \
|
||||
--force
|
||||
```
|
||||
|
||||
```bash
|
||||
|
||||
@@ -95,11 +95,10 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
|
||||
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
|
||||
```
|
||||
- [uv](https://github.com/astral-sh/uv) (for Python dependency management)
|
||||
- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
|
||||
- [node](https://github.com/nodejs/node) (for building the dashboard)
|
||||
|
||||
```bash
|
||||
brew install uv macmon node
|
||||
brew install uv node
|
||||
```
|
||||
- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
|
||||
|
||||
@@ -107,6 +106,17 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
|
||||
rustup toolchain install nightly
|
||||
```
|
||||
- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
|
||||
|
||||
Install the pinned fork revision used by this repo instead of Homebrew `macmon`.
|
||||
Homebrew `macmon 0.6.1` still crashes on Apple M5.
|
||||
|
||||
```bash
|
||||
cargo install --git https://github.com/swiftraccoon/macmon \
|
||||
--rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
|
||||
macmon \
|
||||
--force
|
||||
```
|
||||
|
||||
Clone the repo, build the dashboard, and run exo:
|
||||
|
||||
|
||||
@@ -72,7 +72,7 @@
|
||||
];
|
||||
|
||||
perSystem =
|
||||
{ config, self', inputs', pkgs, lib, system, ... }:
|
||||
{ config, self', pkgs, lib, system, ... }:
|
||||
let
|
||||
# Use pinned nixpkgs for swift-format (swift is broken on x86_64-linux in newer nixpkgs)
|
||||
pkgsSwift = import inputs.nixpkgs-swift { inherit system; };
|
||||
@@ -84,6 +84,17 @@
|
||||
config.allowUnfreePredicate = pkg: (pkg.pname or "") == "metal-toolchain";
|
||||
overlays = [
|
||||
(import ./nix/apple-sdk-overlay.nix)
|
||||
(final: prev: {
|
||||
macmon = prev.macmon.overrideAttrs (_: {
|
||||
version = "git";
|
||||
src = final.fetchFromGitHub {
|
||||
owner = "swiftraccoon";
|
||||
repo = "macmon";
|
||||
rev = "9154d234f763fbeffdcb4135d0bbbaf80609699b";
|
||||
hash = "sha256-CwhilKNbs5XL9/tF5DMwyPBlE/hpmjGNTuxQ36sM50M=";
|
||||
};
|
||||
});
|
||||
})
|
||||
];
|
||||
};
|
||||
treefmt = {
|
||||
|
||||
@@ -71,7 +71,9 @@ MACMON_PATH = shutil.which("macmon")
|
||||
if MACMON_PATH is None:
|
||||
raise SystemExit(
|
||||
"macmon binary not found in PATH. "
|
||||
"Install it via: brew install macmon"
|
||||
"Install the pinned fork used by exo via: "
|
||||
"cargo install --git https://github.com/swiftraccoon/macmon "
|
||||
"--rev 9154d234f763fbeffdcb4135d0bbbaf80609699b macmon --force"
|
||||
)
|
||||
|
||||
BINARIES: list[tuple[str, str]] = [
|
||||
@@ -120,4 +122,3 @@ coll = COLLECT(
|
||||
upx_exclude=[],
|
||||
name="exo",
|
||||
)
|
||||
|
||||
|
||||
@@ -288,7 +288,7 @@ class ThunderboltBridgeInfo(TaggedModel):
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to gather Thunderbolt Bridge info: {e}")
|
||||
logger.opt(exception=e).warning("Failed to gather Thunderbolt Bridge info")
|
||||
return None
|
||||
|
||||
|
||||
@@ -382,18 +382,61 @@ class InfoGatherer:
|
||||
rdma_ctl_poll_interval: float | None = 10 if IS_DARWIN else None
|
||||
disk_poll_interval: float | None = 30
|
||||
_tg: TaskGroup = field(init=False, default_factory=TaskGroup)
|
||||
_psutil_memory_fallback_enabled: bool = field(init=False, default=False)
|
||||
|
||||
def _get_macmon_path(self) -> str | None:
|
||||
return os.getenv("EXO_MACMON_PATH") or shutil.which("macmon")
|
||||
|
||||
async def _can_read_macmon_metrics(self, macmon_path: str) -> bool:
|
||||
try:
|
||||
with fail_after(5):
|
||||
proc = await anyio.run_process(
|
||||
[macmon_path, "pipe", "--samples", "1", "--interval", "100"],
|
||||
check=False,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.opt(exception=e).warning(
|
||||
f"Failed to validate macmon at {macmon_path}"
|
||||
)
|
||||
return False
|
||||
|
||||
if proc.returncode != 0:
|
||||
stderr = proc.stderr.decode("utf-8", errors="replace").strip()
|
||||
logger.warning(
|
||||
f"macmon preflight failed with return code {proc.returncode}: "
|
||||
f"{stderr or 'no stderr'}"
|
||||
)
|
||||
return False
|
||||
|
||||
stdout = proc.stdout.decode("utf-8", errors="replace").strip()
|
||||
if not stdout:
|
||||
logger.warning("macmon preflight returned no metrics")
|
||||
return False
|
||||
|
||||
try:
|
||||
MacmonMetrics.from_raw_json(stdout.splitlines()[0])
|
||||
except ValidationError as e:
|
||||
logger.opt(exception=e).warning(
|
||||
"macmon preflight returned unexpected metrics JSON"
|
||||
)
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
async def run(self):
|
||||
async with self._tg as tg:
|
||||
if IS_DARWIN:
|
||||
if (macmon_path := shutil.which("macmon")) is not None:
|
||||
tg.start_soon(self._monitor_macmon, macmon_path)
|
||||
if (macmon_path := self._get_macmon_path()) is not None:
|
||||
if await self._can_read_macmon_metrics(macmon_path):
|
||||
tg.start_soon(self._monitor_macmon, macmon_path)
|
||||
else:
|
||||
logger.warning(
|
||||
f"macmon at {macmon_path} is unusable, falling back to psutil memory monitoring"
|
||||
)
|
||||
else:
|
||||
# macmon not installed — fall back to psutil for memory
|
||||
logger.warning(
|
||||
"macmon not found, falling back to psutil for memory monitoring"
|
||||
)
|
||||
self.memory_poll_rate = 1
|
||||
tg.start_soon(self._monitor_system_profiler_thunderbolt_data)
|
||||
tg.start_soon(self._monitor_thunderbolt_bridge_status)
|
||||
tg.start_soon(self._monitor_rdma_ctl_status)
|
||||
@@ -418,7 +461,7 @@ class InfoGatherer:
|
||||
with fail_after(30):
|
||||
await self.info_sender.send(await StaticNodeInformation.gather())
|
||||
except Exception as e:
|
||||
logger.warning(f"Error gathering static node info: {e}")
|
||||
logger.opt(exception=e).warning("Error gathering static node info")
|
||||
await anyio.sleep(self.static_info_poll_interval)
|
||||
|
||||
async def _monitor_misc(self):
|
||||
@@ -429,7 +472,7 @@ class InfoGatherer:
|
||||
with fail_after(10):
|
||||
await self.info_sender.send(await MiscData.gather())
|
||||
except Exception as e:
|
||||
logger.warning(f"Error gathering misc data: {e}")
|
||||
logger.opt(exception=e).warning("Error gathering misc data")
|
||||
await anyio.sleep(self.misc_poll_interval)
|
||||
|
||||
async def _monitor_system_profiler_thunderbolt_data(self):
|
||||
@@ -456,7 +499,7 @@ class InfoGatherer:
|
||||
conns = [it for i in data if (it := i.conn()) is not None]
|
||||
await self.info_sender.send(MacThunderboltConnections(conns=conns))
|
||||
except Exception as e:
|
||||
logger.warning(f"Error gathering Thunderbolt data: {e}")
|
||||
logger.opt(exception=e).warning("Error gathering Thunderbolt data")
|
||||
await anyio.sleep(self.system_profiler_interval)
|
||||
|
||||
async def _monitor_memory_usage(self):
|
||||
@@ -474,7 +517,7 @@ class InfoGatherer:
|
||||
MemoryUsage.from_psutil(override_memory=override_memory)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error gathering memory usage: {e}")
|
||||
logger.opt(exception=e).warning("Error gathering memory usage")
|
||||
await anyio.sleep(self.memory_poll_rate)
|
||||
|
||||
async def _watch_system_info(self):
|
||||
@@ -486,7 +529,7 @@ class InfoGatherer:
|
||||
nics = await get_network_interfaces()
|
||||
await self.info_sender.send(NodeNetworkInterfaces(ifaces=nics))
|
||||
except Exception as e:
|
||||
logger.warning(f"Error gathering network interfaces: {e}")
|
||||
logger.opt(exception=e).warning("Error gathering network interfaces")
|
||||
await anyio.sleep(self.interface_watcher_interval)
|
||||
|
||||
async def _monitor_thunderbolt_bridge_status(self):
|
||||
@@ -499,7 +542,9 @@ class InfoGatherer:
|
||||
if curr is not None:
|
||||
await self.info_sender.send(curr)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error gathering Thunderbolt Bridge status: {e}")
|
||||
logger.opt(exception=e).warning(
|
||||
"Error gathering Thunderbolt Bridge status"
|
||||
)
|
||||
await anyio.sleep(self.thunderbolt_bridge_poll_interval)
|
||||
|
||||
async def _monitor_rdma_ctl_status(self):
|
||||
@@ -511,7 +556,7 @@ class InfoGatherer:
|
||||
if curr is not None:
|
||||
await self.info_sender.send(curr)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error gathering RDMA ctl status: {e}")
|
||||
logger.opt(exception=e).warning("Error gathering RDMA ctl status")
|
||||
await anyio.sleep(self.rdma_ctl_poll_interval)
|
||||
|
||||
async def _monitor_disk_usage(self):
|
||||
@@ -522,7 +567,7 @@ class InfoGatherer:
|
||||
with fail_after(5):
|
||||
await self.info_sender.send(await NodeDiskUsage.gather())
|
||||
except Exception as e:
|
||||
logger.warning(f"Error gathering disk usage: {e}")
|
||||
logger.opt(exception=e).warning("Error gathering disk usage")
|
||||
await anyio.sleep(self.disk_poll_interval)
|
||||
|
||||
async def _monitor_macmon(self, macmon_path: str):
|
||||
@@ -554,6 +599,8 @@ class InfoGatherer:
|
||||
logger.warning(
|
||||
f"MacMon produced no output for {read_timeout}s, restarting"
|
||||
)
|
||||
self.memory_poll_rate = 1
|
||||
self._tg.start_soon(self._monitor_memory_usage)
|
||||
except CalledProcessError as e:
|
||||
stderr_msg = "no stderr"
|
||||
stderr_output = cast(bytes | str | None, e.stderr)
|
||||
@@ -566,6 +613,10 @@ class InfoGatherer:
|
||||
logger.warning(
|
||||
f"MacMon failed with return code {e.returncode}: {stderr_msg}"
|
||||
)
|
||||
self.memory_poll_rate = 1
|
||||
self._tg.start_soon(self._monitor_memory_usage)
|
||||
except Exception as e:
|
||||
logger.warning(f"Error in macmon monitor: {e}")
|
||||
logger.opt(exception=e).warning("Error in macmon monitor")
|
||||
self.memory_poll_rate = 1
|
||||
self._tg.start_soon(self._monitor_memory_usage)
|
||||
await anyio.sleep(self.macmon_interval)
|
||||
|
||||
Reference in New Issue
Block a user