override macmon in flake (#1747)

updates macmon to an upstream fork that fixes m5 max issues.
might see if the upstream version gets merged before we release.

---------

Co-authored-by: Alex Cheema <alexcheema123@gmail.com>

CONTRIBUTING.md

@@ -11,9 +11,18 @@ To run EXO from source:
   ```bash
   brew install uv
   ```
-- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+- [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
   ```bash
-  brew install macmon
+  curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
+  rustup toolchain install nightly
+  ```
+- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+  Use the pinned fork revision used by this repo instead of Homebrew `macmon`.
+  ```bash
+  cargo install --git https://github.com/swiftraccoon/macmon \
+    --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
+    macmon \
+    --force
   ```
 
 ```bash

README.md

@@ -95,11 +95,10 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
   /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
   ```
 - [uv](https://github.com/astral-sh/uv) (for Python dependency management)
-- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
 - [node](https://github.com/nodejs/node) (for building the dashboard)
 
   ```bash
-  brew install uv macmon node
+  brew install uv node
   ```
 - [rust](https://github.com/rust-lang/rustup) (to build Rust bindings, nightly for now)
 
@@ -107,6 +106,17 @@ Then restart the Nix daemon: `sudo launchctl kickstart -k system/org.nixos.nix-d
   curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
   rustup toolchain install nightly
   ```
+- [macmon](https://github.com/vladkens/macmon) (for hardware monitoring on Apple Silicon)
+
+  Install the pinned fork revision used by this repo instead of Homebrew `macmon`.
+  Homebrew `macmon 0.6.1` still crashes on Apple M5.
+
+  ```bash
+  cargo install --git https://github.com/swiftraccoon/macmon \
+    --rev 9154d234f763fbeffdcb4135d0bbbaf80609699b \
+    macmon \
+    --force
+  ```
 
 Clone the repo, build the dashboard, and run exo:
 

flake.nix

@@ -72,7 +72,7 @@
       ];
 
       perSystem =
-        { config, self', inputs', pkgs, lib, system, ... }:
+        { config, self', pkgs, lib, system, ... }:
         let
           # Use pinned nixpkgs for swift-format (swift is broken on x86_64-linux in newer nixpkgs)
           pkgsSwift = import inputs.nixpkgs-swift { inherit system; };
@@ -84,6 +84,17 @@
             config.allowUnfreePredicate = pkg: (pkg.pname or "") == "metal-toolchain";
             overlays = [
               (import ./nix/apple-sdk-overlay.nix)
+              (final: prev: {
+                macmon = prev.macmon.overrideAttrs (_: {
+                  version = "git";
+                  src = final.fetchFromGitHub {
+                    owner = "swiftraccoon";
+                    repo = "macmon";
+                    rev = "9154d234f763fbeffdcb4135d0bbbaf80609699b";
+                    hash = "sha256-CwhilKNbs5XL9/tF5DMwyPBlE/hpmjGNTuxQ36sM50M=";
+                  };
+                });
+              })
             ];
           };
           treefmt = {

packaging/pyinstaller/exo.spec

@@ -71,7 +71,9 @@ MACMON_PATH = shutil.which("macmon")
 if MACMON_PATH is None:
     raise SystemExit(
         "macmon binary not found in PATH. "
-        "Install it via: brew install macmon"
+        "Install the pinned fork used by exo via: "
+        "cargo install --git https://github.com/swiftraccoon/macmon "
+        "--rev 9154d234f763fbeffdcb4135d0bbbaf80609699b macmon --force"
     )
 
 BINARIES: list[tuple[str, str]] = [
@@ -120,4 +122,3 @@ coll = COLLECT(
     upx_exclude=[],
     name="exo",
 )
-

src/exo/utils/info_gatherer/info_gatherer.py

@@ -288,7 +288,7 @@ class ThunderboltBridgeInfo(TaggedModel):
                 )
             )
         except Exception as e:
-            logger.warning(f"Failed to gather Thunderbolt Bridge info: {e}")
+            logger.opt(exception=e).warning("Failed to gather Thunderbolt Bridge info")
             return None
 
 
@@ -382,18 +382,61 @@ class InfoGatherer:
     rdma_ctl_poll_interval: float | None = 10 if IS_DARWIN else None
     disk_poll_interval: float | None = 30
     _tg: TaskGroup = field(init=False, default_factory=TaskGroup)
+    _psutil_memory_fallback_enabled: bool = field(init=False, default=False)
+
+    def _get_macmon_path(self) -> str | None:
+        return os.getenv("EXO_MACMON_PATH") or shutil.which("macmon")
+
+    async def _can_read_macmon_metrics(self, macmon_path: str) -> bool:
+        try:
+            with fail_after(5):
+                proc = await anyio.run_process(
+                    [macmon_path, "pipe", "--samples", "1", "--interval", "100"],
+                    check=False,
+                )
+        except Exception as e:
+            logger.opt(exception=e).warning(
+                f"Failed to validate macmon at {macmon_path}"
+            )
+            return False
+
+        if proc.returncode != 0:
+            stderr = proc.stderr.decode("utf-8", errors="replace").strip()
+            logger.warning(
+                f"macmon preflight failed with return code {proc.returncode}: "
+                f"{stderr or 'no stderr'}"
+            )
+            return False
+
+        stdout = proc.stdout.decode("utf-8", errors="replace").strip()
+        if not stdout:
+            logger.warning("macmon preflight returned no metrics")
+            return False
+
+        try:
+            MacmonMetrics.from_raw_json(stdout.splitlines()[0])
+        except ValidationError as e:
+            logger.opt(exception=e).warning(
+                "macmon preflight returned unexpected metrics JSON"
+            )
+            return False
+
+        return True
 
     async def run(self):
         async with self._tg as tg:
             if IS_DARWIN:
-                if (macmon_path := shutil.which("macmon")) is not None:
+                if (macmon_path := self._get_macmon_path()) is not None:
-                    tg.start_soon(self._monitor_macmon, macmon_path)
+                    if await self._can_read_macmon_metrics(macmon_path):
+                        tg.start_soon(self._monitor_macmon, macmon_path)
+                    else:
+                        logger.warning(
+                            f"macmon at {macmon_path} is unusable, falling back to psutil memory monitoring"
+                        )
                 else:
-                    # macmon not installed — fall back to psutil for memory
                     logger.warning(
                         "macmon not found, falling back to psutil for memory monitoring"
                     )
-                    self.memory_poll_rate = 1
                 tg.start_soon(self._monitor_system_profiler_thunderbolt_data)
                 tg.start_soon(self._monitor_thunderbolt_bridge_status)
                 tg.start_soon(self._monitor_rdma_ctl_status)
@@ -418,7 +461,7 @@ class InfoGatherer:
                 with fail_after(30):
                     await self.info_sender.send(await StaticNodeInformation.gather())
             except Exception as e:
-                logger.warning(f"Error gathering static node info: {e}")
+                logger.opt(exception=e).warning("Error gathering static node info")
             await anyio.sleep(self.static_info_poll_interval)
 
     async def _monitor_misc(self):
@@ -429,7 +472,7 @@ class InfoGatherer:
                 with fail_after(10):
                     await self.info_sender.send(await MiscData.gather())
             except Exception as e:
-                logger.warning(f"Error gathering misc data: {e}")
+                logger.opt(exception=e).warning("Error gathering misc data")
             await anyio.sleep(self.misc_poll_interval)
 
     async def _monitor_system_profiler_thunderbolt_data(self):
@@ -456,7 +499,7 @@ class InfoGatherer:
                     conns = [it for i in data if (it := i.conn()) is not None]
                     await self.info_sender.send(MacThunderboltConnections(conns=conns))
             except Exception as e:
-                logger.warning(f"Error gathering Thunderbolt data: {e}")
+                logger.opt(exception=e).warning("Error gathering Thunderbolt data")
             await anyio.sleep(self.system_profiler_interval)
 
     async def _monitor_memory_usage(self):
@@ -474,7 +517,7 @@ class InfoGatherer:
                     MemoryUsage.from_psutil(override_memory=override_memory)
                 )
             except Exception as e:
-                logger.warning(f"Error gathering memory usage: {e}")
+                logger.opt(exception=e).warning("Error gathering memory usage")
             await anyio.sleep(self.memory_poll_rate)
 
     async def _watch_system_info(self):
@@ -486,7 +529,7 @@ class InfoGatherer:
                     nics = await get_network_interfaces()
                     await self.info_sender.send(NodeNetworkInterfaces(ifaces=nics))
             except Exception as e:
-                logger.warning(f"Error gathering network interfaces: {e}")
+                logger.opt(exception=e).warning("Error gathering network interfaces")
             await anyio.sleep(self.interface_watcher_interval)
 
     async def _monitor_thunderbolt_bridge_status(self):
@@ -499,7 +542,9 @@ class InfoGatherer:
                     if curr is not None:
                         await self.info_sender.send(curr)
             except Exception as e:
-                logger.warning(f"Error gathering Thunderbolt Bridge status: {e}")
+                logger.opt(exception=e).warning(
+                    "Error gathering Thunderbolt Bridge status"
+                )
             await anyio.sleep(self.thunderbolt_bridge_poll_interval)
 
     async def _monitor_rdma_ctl_status(self):
@@ -511,7 +556,7 @@ class InfoGatherer:
                 if curr is not None:
                     await self.info_sender.send(curr)
             except Exception as e:
-                logger.warning(f"Error gathering RDMA ctl status: {e}")
+                logger.opt(exception=e).warning("Error gathering RDMA ctl status")
             await anyio.sleep(self.rdma_ctl_poll_interval)
 
     async def _monitor_disk_usage(self):
@@ -522,7 +567,7 @@ class InfoGatherer:
                 with fail_after(5):
                     await self.info_sender.send(await NodeDiskUsage.gather())
             except Exception as e:
-                logger.warning(f"Error gathering disk usage: {e}")
+                logger.opt(exception=e).warning("Error gathering disk usage")
             await anyio.sleep(self.disk_poll_interval)
 
     async def _monitor_macmon(self, macmon_path: str):
@@ -554,6 +599,8 @@ class InfoGatherer:
                 logger.warning(
                     f"MacMon produced no output for {read_timeout}s, restarting"
                 )
+                self.memory_poll_rate = 1
+                self._tg.start_soon(self._monitor_memory_usage)
             except CalledProcessError as e:
                 stderr_msg = "no stderr"
                 stderr_output = cast(bytes | str | None, e.stderr)
@@ -566,6 +613,10 @@ class InfoGatherer:
                 logger.warning(
                     f"MacMon failed with return code {e.returncode}: {stderr_msg}"
                 )
+                self.memory_poll_rate = 1
+                self._tg.start_soon(self._monitor_memory_usage)
             except Exception as e:
-                logger.warning(f"Error in macmon monitor: {e}")
+                logger.opt(exception=e).warning("Error in macmon monitor")
+                self.memory_poll_rate = 1
+                self._tg.start_soon(self._monitor_memory_usage)
             await anyio.sleep(self.macmon_interval)