diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20f6f9f9..cb6d78df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,9 +11,18 @@ To run EXO from source: ```bash brew install uv ``` -- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon) +- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now) ```bash - brew install macmon + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh + rustup toolchain install nightly + ``` +- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon) + Use the pinned fork revision used by this repo instead of Homebrew `macmon`. + ```bash + cargo install --git https://github.com/swiftraccoon/macmon \ + --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \ + macmon \ + --force ``` ```bash diff --git a/README.md b/README.md index f1cf9f39..5104f14f 100644 --- a/README.md +++ b/README.md @@ -95,11 +95,10 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" ``` - [uv](https://github.com/astral-sh/uv) (for Python dependency management) -- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon) - [node](https://github.com/nodejs/node) (for building the dashboard) ```bash - brew install uv macmon node + brew install uv node ``` - [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now) @@ -107,6 +106,17 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh rustup toolchain install nightly ``` +- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon) + + Install the pinned fork revision used by this repo instead of Homebrew `macmon`. + Homebrew `macmon 0.6.1` still crashes on Apple M5. + + ```bash + cargo install --git https://github.com/swiftraccoon/macmon \ + --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \ + macmon \ + --force + ``` Clone the repo, build the dashboard, and run exo: diff --git a/flake.nix b/flake.nix index ff5ec712..0220d44d 100644 --- a/flake.nix +++ b/flake.nix @@ -72,7 +72,7 @@ ]; perSystem = - { config, self', inputs', pkgs, lib, system, ... }: + { config, self', pkgs, lib, system, ... }: let # Use pinned nixpkgs for swift-format (swift is broken on x86_64-linux in newer nixpkgs) pkgsSwift = import inputs.nixpkgs-swift { inherit system; }; @@ -84,6 +84,17 @@ config.allowUnfreePredicate = pkg: (pkg.pname or "") == "metal-toolchain"; overlays = [ (import ./nix/apple-sdk-overlay.nix) + (final: prev: { + macmon = prev.macmon.overrideAttrs (_: { + version = "git"; + src = final.fetchFromGitHub { + owner = "swiftraccoon"; + repo = "macmon"; + rev = "9154d234f763fbeffdcb4135d0bbbaf80609699b"; + hash = "sha256-CwhilKNbs5XL9/tF5DMwyPBlE/hpmjGNTuxQ36sM50M="; + }; + }); + }) ]; }; treefmt = { diff --git a/packaging/pyinstaller/exo.spec b/packaging/pyinstaller/exo.spec index f991447e..143d0ea0 100644 --- a/packaging/pyinstaller/exo.spec +++ b/packaging/pyinstaller/exo.spec @@ -71,7 +71,9 @@ MACMON_PATH = shutil.which("macmon") if MACMON_PATH is None: raise SystemExit( "macmon binary not found in PATH. " - "Install it via: brew install macmon" + "Install the pinned fork used by exo via: " + "cargo install --git https://github.com/swiftraccoon/macmon " + "--rev 9154d234f763fbeffdcb4135d0bbbaf80609699b macmon --force" ) BINARIES: list[tuple[str, str]] = [ @@ -120,4 +122,3 @@ coll = COLLECT( upx_exclude=[], name="exo", ) - diff --git a/src/exo/utils/info_gatherer/info_gatherer.py b/src/exo/utils/info_gatherer/info_gatherer.py index b5e5d24e..52276e64 100644 --- a/src/exo/utils/info_gatherer/info_gatherer.py +++ b/src/exo/utils/info_gatherer/info_gatherer.py @@ -288,7 +288,7 @@ class ThunderboltBridgeInfo(TaggedModel): ) ) except Exception as e: - logger.warning(f"Failed to gather Thunderbolt Bridge info: {e}") + logger.opt(exception=e).warning("Failed to gather Thunderbolt Bridge info") return None @@ -382,18 +382,61 @@ class InfoGatherer: rdma_ctl_poll_interval: float | None = 10 if IS_DARWIN else None disk_poll_interval: float | None = 30 _tg: TaskGroup = field(init=False, default_factory=TaskGroup) + _psutil_memory_fallback_enabled: bool = field(init=False, default=False) + + def _get_macmon_path(self) -> str | None: + return os.getenv("EXO_MACMON_PATH") or shutil.which("macmon") + + async def _can_read_macmon_metrics(self, macmon_path: str) -> bool: + try: + with fail_after(5): + proc = await anyio.run_process( + [macmon_path, "pipe", "--samples", "1", "--interval", "100"], + check=False, + ) + except Exception as e: + logger.opt(exception=e).warning( + f"Failed to validate macmon at {macmon_path}" + ) + return False + + if proc.returncode != 0: + stderr = proc.stderr.decode("utf-8", errors="replace").strip() + logger.warning( + f"macmon preflight failed with return code {proc.returncode}: " + f"{stderr or 'no stderr'}" + ) + return False + + stdout = proc.stdout.decode("utf-8", errors="replace").strip() + if not stdout: + logger.warning("macmon preflight returned no metrics") + return False + + try: + MacmonMetrics.from_raw_json(stdout.splitlines()[0]) + except ValidationError as e: + logger.opt(exception=e).warning( + "macmon preflight returned unexpected metrics JSON" + ) + return False + + return True async def run(self): async with self._tg as tg: if IS_DARWIN: - if (macmon_path := shutil.which("macmon")) is not None: - tg.start_soon(self._monitor_macmon, macmon_path) + if (macmon_path := self._get_macmon_path()) is not None: + if await self._can_read_macmon_metrics(macmon_path): + tg.start_soon(self._monitor_macmon, macmon_path) + else: + logger.warning( + f"macmon at {macmon_path} is unusable, falling back to psutil memory monitoring" + ) else: - # macmon not installed — fall back to psutil for memory logger.warning( "macmon not found, falling back to psutil for memory monitoring" ) - self.memory_poll_rate = 1 tg.start_soon(self._monitor_system_profiler_thunderbolt_data) tg.start_soon(self._monitor_thunderbolt_bridge_status) tg.start_soon(self._monitor_rdma_ctl_status) @@ -418,7 +461,7 @@ class InfoGatherer: with fail_after(30): await self.info_sender.send(await StaticNodeInformation.gather()) except Exception as e: - logger.warning(f"Error gathering static node info: {e}") + logger.opt(exception=e).warning("Error gathering static node info") await anyio.sleep(self.static_info_poll_interval) async def _monitor_misc(self): @@ -429,7 +472,7 @@ class InfoGatherer: with fail_after(10): await self.info_sender.send(await MiscData.gather()) except Exception as e: - logger.warning(f"Error gathering misc data: {e}") + logger.opt(exception=e).warning("Error gathering misc data") await anyio.sleep(self.misc_poll_interval) async def _monitor_system_profiler_thunderbolt_data(self): @@ -456,7 +499,7 @@ class InfoGatherer: conns = [it for i in data if (it := i.conn()) is not None] await self.info_sender.send(MacThunderboltConnections(conns=conns)) except Exception as e: - logger.warning(f"Error gathering Thunderbolt data: {e}") + logger.opt(exception=e).warning("Error gathering Thunderbolt data") await anyio.sleep(self.system_profiler_interval) async def _monitor_memory_usage(self): @@ -474,7 +517,7 @@ class InfoGatherer: MemoryUsage.from_psutil(override_memory=override_memory) ) except Exception as e: - logger.warning(f"Error gathering memory usage: {e}") + logger.opt(exception=e).warning("Error gathering memory usage") await anyio.sleep(self.memory_poll_rate) async def _watch_system_info(self): @@ -486,7 +529,7 @@ class InfoGatherer: nics = await get_network_interfaces() await self.info_sender.send(NodeNetworkInterfaces(ifaces=nics)) except Exception as e: - logger.warning(f"Error gathering network interfaces: {e}") + logger.opt(exception=e).warning("Error gathering network interfaces") await anyio.sleep(self.interface_watcher_interval) async def _monitor_thunderbolt_bridge_status(self): @@ -499,7 +542,9 @@ class InfoGatherer: if curr is not None: await self.info_sender.send(curr) except Exception as e: - logger.warning(f"Error gathering Thunderbolt Bridge status: {e}") + logger.opt(exception=e).warning( + "Error gathering Thunderbolt Bridge status" + ) await anyio.sleep(self.thunderbolt_bridge_poll_interval) async def _monitor_rdma_ctl_status(self): @@ -511,7 +556,7 @@ class InfoGatherer: if curr is not None: await self.info_sender.send(curr) except Exception as e: - logger.warning(f"Error gathering RDMA ctl status: {e}") + logger.opt(exception=e).warning("Error gathering RDMA ctl status") await anyio.sleep(self.rdma_ctl_poll_interval) async def _monitor_disk_usage(self): @@ -522,7 +567,7 @@ class InfoGatherer: with fail_after(5): await self.info_sender.send(await NodeDiskUsage.gather()) except Exception as e: - logger.warning(f"Error gathering disk usage: {e}") + logger.opt(exception=e).warning("Error gathering disk usage") await anyio.sleep(self.disk_poll_interval) async def _monitor_macmon(self, macmon_path: str): @@ -554,6 +599,8 @@ class InfoGatherer: logger.warning( f"MacMon produced no output for {read_timeout}s, restarting" ) + self.memory_poll_rate = 1 + self._tg.start_soon(self._monitor_memory_usage) except CalledProcessError as e: stderr_msg = "no stderr" stderr_output = cast(bytes | str | None, e.stderr) @@ -566,6 +613,10 @@ class InfoGatherer: logger.warning( f"MacMon failed with return code {e.returncode}: {stderr_msg}" ) + self.memory_poll_rate = 1 + self._tg.start_soon(self._monitor_memory_usage) except Exception as e: - logger.warning(f"Error in macmon monitor: {e}") + logger.opt(exception=e).warning("Error in macmon monitor") + self.memory_poll_rate = 1 + self._tg.start_soon(self._monitor_memory_usage) await anyio.sleep(self.macmon_interval)