From 8392e78afe152d6311f5465f70d7ae366fc9902c Mon Sep 17 00:00:00 2001 From: Jake Hillion Date: Tue, 17 Feb 2026 10:52:05 +0000 Subject: [PATCH] bench: add spec for automatic canary benchmarks (#1483) Adds all the models that can fit onto a single M3 Ultra for single machine benchmarks. Fixes the macOS version, GPU spec, and chip type for maximum reproducibility. Specifies the minimum memory accordingly for each type of model, using the smallest machine available (the smallest M3 Ultra is 96GiB). Test plan: - Running this with some code that makes machines of this spec available and stores the results. It works. This will become part of a larger testing/stability strategy once we've collected more of the data. --- bench/bench.toml | 7 ++ bench/single-m3-ultra.toml | 189 +++++++++++++++++++++++++++++++++++++ 2 files changed, 196 insertions(+) create mode 100644 bench/bench.toml create mode 100644 bench/single-m3-ultra.toml diff --git a/bench/bench.toml b/bench/bench.toml new file mode 100644 index 00000000..3b3c8c07 --- /dev/null +++ b/bench/bench.toml @@ -0,0 +1,7 @@ +# Canary benchmark manifest +# +# Lists the suite files to include. Each file defines benchmarks +# with shared constraints, topology, and default args. +include = [ + "single-m3-ultra.toml", +] diff --git a/bench/single-m3-ultra.toml b/bench/single-m3-ultra.toml new file mode 100644 index 00000000..1d167ac9 --- /dev/null +++ b/bench/single-m3-ultra.toml @@ -0,0 +1,189 @@ +# Single-node M3 Ultra benchmarks +# +# Shared constraints applied to ALL benchmarks in this file. +constraints = [ + "All(MacOsBuild(=25D125))", + "Hosts(=1)", + "All(Chip(m3_ultra))", + "All(GpuCores(=80))", +] + +[topology] +type = "none" + +# Default args merged into each benchmark's args (benchmark-level args win). +[defaults] +pp = [512, 2048, 8192, 16384] +tg = 128 + +[[benchmark]] +model = "mlx-community/Meta-Llama-3.1-70B-Instruct-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/gpt-oss-120b-MXFP4-Q8" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.7-Flash-8bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Coder-Next-6bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-30B-A3B-8bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-0.6B-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-0.6B-8bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Llama-3.2-1B-Instruct-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Llama-3.2-3B-Instruct-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Llama-3.2-3B-Instruct-8bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Meta-Llama-3.1-8B-Instruct-8bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Meta-Llama-3.1-8B-Instruct-bf16" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/gpt-oss-20b-MXFP4-Q8" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-30B-A3B-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.7-Flash-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.7-Flash-5bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.7-Flash-6bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Llama-3.3-70B-Instruct-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Coder-Next-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Coder-Next-5bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Coder-Next-8bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit" +extra_constraints = ["All(Memory(>=96GiB))"] + +[[benchmark]] +model = "mlx-community/Llama-3.3-70B-Instruct-8bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/llama-3.3-70b-instruct-fp16" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.5-Air-8bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.5-Air-bf16" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.7-4bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/MiniMax-M2.1-3bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/MiniMax-M2.1-8bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Coder-Next-bf16" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/Step-3.5-Flash-4bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/Step-3.5-Flash-6bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/Step-3.5-Flash-8Bit" +extra_constraints = ["All(Memory(>=256GiB))"] + +[[benchmark]] +model = "mlx-community/DeepSeek-V3.1-4bit" +extra_constraints = ["All(Memory(>=512GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.7-6bit" +extra_constraints = ["All(Memory(>=512GiB))"] + +[[benchmark]] +model = "mlx-community/GLM-4.7-8bit-gs32" +extra_constraints = ["All(Memory(>=512GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit" +extra_constraints = ["All(Memory(>=512GiB))"] + +[[benchmark]] +model = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit" +extra_constraints = ["All(Memory(>=512GiB))"]