woahg

eprint banner
our banner was being printed to stdout but should be printed to stderr as its essentially a log message
2026-02-17 11:44:23 +00:00 · 2026-02-17 11:43:06 +00:00 · 2026-02-17 10:52:05 +00:00 · 2026-02-16 19:26:19 +00:00
80 changed files with 1045 additions and 6593 deletions
@@ -1,15 +0,0 @@
-.venv/
-.direnv/
-target/
-.git/
-.idea/
-.pytest_cache/
-.ruff_cache/
-dashboard/node_modules/
-dashboard/.svelte-kit/
-dashboard/build/
-dist/
-*.pdb
-**/__pycache__
-**/.DS_Store
-.mlx_typings/
@@ -1,44 +0,0 @@
-name: e2e-tests
-
-on:
-  push:
-    branches:
-      - e2e-tests
-  pull_request:
-    branches:
-      - staging
-      - main
-
-jobs:
-  e2e:
-    runs-on: ubuntu-latest
-    timeout-minutes: 45
-    steps:
-      - name: Free up disk space
-        run: |
-          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \
-            /opt/hostedtoolcache /usr/local/share/boost /usr/share/swift \
-            /opt/microsoft /opt/az
-          docker system prune -af
-          df -h /
-
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          lfs: false
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build E2E image with cache
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: e2e/Dockerfile
-          tags: exo-e2e:latest
-          load: true
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      - name: Run E2E tests
-        run: python3 e2e/run_all.py
@@ -141,12 +141,6 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb"

-[[package]]
-name = "arrayvec"
-version = "0.7.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
-
 [[package]]
 name = "asn1-rs"
 version = "0.7.1"
@@ -304,19 +298,6 @@ version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba"

-[[package]]
-name = "bigdecimal"
-version = "0.4.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934"
-dependencies = [
- "autocfg",
- "libm",
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
 [[package]]
 name = "bimap"
 version = "0.6.3"
@@ -516,15 +497,6 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3"

-[[package]]
-name = "convert_case"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9"
-dependencies = [
- "unicode-segmentation",
-]
-
 [[package]]
 name = "core-foundation"
 version = "0.9.4"
@@ -701,17 +673,6 @@ dependencies = [
 "syn 2.0.111",
 ]

-[[package]]
-name = "delegate"
-version = "0.13.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.111",
-]
-
 [[package]]
 name = "der"
 version = "0.7.10"
@@ -746,29 +707,6 @@ dependencies = [
 "powerfmt",
 ]

-[[package]]
-name = "derive_more"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618"
-dependencies = [
- "derive_more-impl",
-]
-
-[[package]]
-name = "derive_more-impl"
-version = "2.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b"
-dependencies = [
- "convert_case",
- "proc-macro2",
- "quote",
- "rustc_version",
- "syn 2.0.111",
- "unicode-xid",
-]
-
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -938,37 +876,23 @@ dependencies = [
 name = "exo_pyo3_bindings"
 version = "0.0.1"
 dependencies = [
- "delegate",
- "derive_more",
 "env_logger",
- "extend",
- "futures",
- "impl-trait-for-tuples",
+ "futures-lite",
 "libp2p",
 "log",
 "networking",
- "once_cell",
- "pin-project",
 "pyo3",
 "pyo3-async-runtimes",
 "pyo3-log",
 "pyo3-stub-gen",
- "thiserror 2.0.17",
- "thread_local",
 "tokio",
- "util",
 ]

 [[package]]
-name = "extend"
-version = "1.2.0"
+name = "fastrand"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "311a6d2f1f9d60bff73d2c78a0af97ed27f79672f15c238192a5bbb64db56d00"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.111",
-]
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"

 [[package]]
 name = "ff"
@@ -1078,7 +1002,10 @@ version = "2.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad"
 dependencies = [
+ "fastrand",
 "futures-core",
+ "futures-io",
+ "parking",
 "pin-project-lite",
 ]

@@ -1640,17 +1567,6 @@ dependencies = [
 "xmltree",
 ]

-[[package]]
-name = "impl-trait-for-tuples"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.111",
-]
-
 [[package]]
 name = "indexmap"
 version = "2.12.1"
@@ -1805,12 +1721,6 @@ dependencies = [
 "cpufeatures",
 ]

-[[package]]
-name = "keccak-const"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57d8d8ce877200136358e0bbff3a77965875db3af755a11e1fa6b1b3e2df13ea"
-
 [[package]]
 name = "lalrpop-util"
 version = "0.20.2"
@@ -1829,12 +1739,6 @@ version = "0.2.178"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"

-[[package]]
-name = "libm"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
-
 [[package]]
 name = "libp2p"
 version = "0.56.0"
@@ -2823,20 +2727,10 @@ dependencies = [
 name = "networking"
 version = "0.0.1"
 dependencies = [
- "delegate",
- "derive_more",
- "either",
- "extend",
- "futures",
- "futures-timer",
- "impl-trait-for-tuples",
- "keccak-const",
 "libp2p",
 "log",
- "thiserror 2.0.17",
 "tokio",
 "tracing-subscriber",
- "util",
 ]

 [[package]]
@@ -2918,17 +2812,6 @@ dependencies = [
 "num-traits",
 ]

-[[package]]
-name = "num-rational"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
-dependencies = [
- "num-bigint",
- "num-integer",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -3279,28 +3162,14 @@ version = "0.27.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d"
 dependencies = [
- "bigdecimal",
- "either",
- "hashbrown 0.16.1",
- "indexmap",
 "indoc",
- "inventory",
 "libc",
- "lock_api",
 "memoffset",
- "num-bigint",
- "num-complex",
- "num-rational",
- "num-traits",
 "once_cell",
- "ordered-float",
- "parking_lot",
 "portable-atomic",
 "pyo3-build-config",
 "pyo3-ffi",
 "pyo3-macros",
- "rust_decimal",
- "smallvec",
 "unindent",
 ]

@@ -3741,16 +3610,6 @@ dependencies = [
 "tokio",
 ]

-[[package]]
-name = "rust_decimal"
-version = "1.39.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282"
-dependencies = [
- "arrayvec",
- "num-traits",
-]
-
 [[package]]
 name = "rustc-hash"
 version = "1.1.0"
@@ -4615,24 +4474,12 @@ version = "1.0.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"

-[[package]]
-name = "unicode-segmentation"
-version = "1.12.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
-
 [[package]]
 name = "unicode-width"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"

-[[package]]
-name = "unicode-xid"
-version = "0.2.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
-
 [[package]]
 name = "unicode_names2"
 version = "1.3.0"
@@ -4713,10 +4560,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"

-[[package]]
-name = "util"
-version = "0.0.1"
-
 [[package]]
 name = "uuid"
 version = "1.19.0"
@@ -3,7 +3,6 @@ resolver = "3"
 members = [
    "rust/networking",
    "rust/exo_pyo3_bindings",
-    "rust/util",
 ]

 [workspace.package]
@@ -24,62 +23,18 @@ opt-level = 3
 [workspace.dependencies]
 ## Crate members as common dependencies
 networking = { path = "rust/networking" }
-util = { path = "rust/util" }
-
-# Proc-macro authoring tools
-syn = "2.0"
-quote = "1.0"
-proc-macro2 = "1.0"
-darling = "0.20"
-
-# Macro dependecies
-extend = "1.2"
-delegate = "0.13"
-impl-trait-for-tuples = "0.2"
-clap = "4.5"
-derive_more = { version = "2.0.1", features = ["display"] }
-pin-project = "1"
-
-# Utility dependencies
-itertools = "0.14"
-thiserror = "2"
-internment = "0.8"
-recursion = "0.5"
-regex = "1.11"
-once_cell = "1.21"
-thread_local = "1.1"
-bon = "3.4"
-generativity = "1.1"
-anyhow = "1.0"
-keccak-const = "0.2"
-
-# Functional generics/lenses frameworks
-frunk_core = "0.4"
-frunk = "0.4"
-frunk_utils = "0.2"
-frunk-enum-core = "0.3"

 # Async dependencies
 tokio = "1.46"
-futures = "0.3"
-futures-util = "0.3"
-futures-timer = "3.0"
-
-# Data structures
-either = "1.15"
-ordered-float = "5.0"
-ahash = "0.8"

 # Tracing/logging
 log = "0.4"

 # networking
 libp2p = "0.56"
-libp2p-tcp = "0.44"

 [workspace.lints.rust]
-static_mut_refs = "warn"      # Or use "warn" instead of deny
-incomplete_features = "allow"
+static_mut_refs = "warn"

 # Clippy's lint category level configurations;
 # every member crate needs to inherit these by adding
@@ -100,64 +55,3 @@ perf = { level = "warn", priority = -1 }
 pedantic = { level = "warn", priority = -1 }
 nursery = { level = "warn", priority = -1 }
 cargo = { level = "warn", priority = -1 }
-
-# Individual Clippy lints from the `restriction` category
-arithmetic_side_effects = "warn"
-as_conversions = "warn"
-assertions_on_result_states = "warn"
-clone_on_ref_ptr = "warn"
-decimal_literal_representation = "warn"
-default_union_representation = "warn"
-deref_by_slicing = "warn"
-disallowed_script_idents = "deny"
-else_if_without_else = "warn"
-empty_enum_variants_with_brackets = "warn"
-empty_structs_with_brackets = "warn"
-error_impl_error = "warn"
-exit = "deny"
-expect_used = "warn"
-float_cmp_const = "warn"
-get_unwrap = "warn"
-if_then_some_else_none = "warn"
-impl_trait_in_params = "warn"
-indexing_slicing = "warn"
-infinite_loop = "warn"
-let_underscore_must_use = "warn"
-let_underscore_untyped = "warn"
-lossy_float_literal = "warn"
-mem_forget = "warn"
-missing_inline_in_public_items = "warn"
-multiple_inherent_impl = "warn"
-multiple_unsafe_ops_per_block = "warn"
-mutex_atomic = "warn"
-non_zero_suggestions = "warn"
-panic = "warn"
-partial_pub_fields = "warn"
-pattern_type_mismatch = "warn"
-pub_without_shorthand = "warn"
-rc_buffer = "warn"
-rc_mutex = "warn"
-redundant_type_annotations = "warn"
-renamed_function_params = "warn"
-rest_pat_in_fully_bound_structs = "warn"
-same_name_method = "warn"
-self_named_module_files = "deny"
-semicolon_inside_block = "warn"
-shadow_same = "warn"
-shadow_unrelated = "warn"
-str_to_string = "warn"
-string_add = "warn"
-string_lit_chars_any = "warn"
-string_to_string = "warn"
-tests_outside_test_module = "warn"
-todo = "warn"
-try_err = "warn"
-undocumented_unsafe_blocks = "warn"
-unnecessary_safety_comment = "warn"
-unnecessary_safety_doc = "warn"
-unneeded_field_pattern = "warn"
-unseparated_literal_suffix = "warn"
-unused_result_ok = "warn"
-unused_trait_names = "warn"
-unwrap_used = "warn"
-verbose_file_reads = "warn"
@@ -1,5 +1,5 @@
 # Missed things
-[X] Log EXO_LIBP2P_NAMESPACE on start in exo/main.py
+[X] Log namespace on start in exo/main.py
 [X] Ordering of warmup was changed, which is wrong. It was changed to rank < n-1, then rank=n-1. It should be rank!=0 then rank=0 (this matches the auto_parallel implementation. NOTE: we use a different convention to mlx-lm, our terminal rank is rank=n-1 whereas mlx-lm is rank=0 hence i can see why this was changed wrongly).
 [X] Downloads keying by model_id not shard_metadata (worker/plan.py, worker/main.py).
 [X] Fetching download status of all models on start
@@ -199,14 +199,14 @@ The app will ask for permission to modify system settings and install a new Netw

 **Custom Namespace for Cluster Isolation:**

-The macOS app includes a custom namespace feature that allows you to isolate your exo cluster from others on the same network. This is configured through the `EXO_LIBP2P_NAMESPACE` setting:
+The macOS app includes a custom namespace feature that allows you to isolate your exo cluster from others on the same network. This is configured through the `--namespace` cli arg:

 - **Use cases**:
  - Running multiple separate exo clusters on the same network
  - Isolating development/testing clusters from production clusters
  - Preventing accidental cluster joining

- **Configuration**: Access this setting in the app's Advanced settings (or set the `EXO_LIBP2P_NAMESPACE` environment variable when running from source)
+- **Configuration**: Access this setting in the app's Advanced settings (or set the `--namespace` argument when running from source)

 The namespace is logged on startup for debugging purposes.

@@ -418,4 +418,4 @@ On macOS, exo uses the GPU. On Linux, exo currently runs on CPU. We are working

 ## Contributing

-See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to contribute to exo.
+See [CONTRIBUTING.md](CONTRIBUTING.md) for guidelines on how to contribute to exo.
@@ -82,6 +82,7 @@ final class ExoProcessController: ObservableObject {

            let child = Process()
            child.executableURL = executableURL
+            child.arguments = ["--namespace", computeNamespace()]
            let exoHomeURL = Self.exoDirectoryURL
            try? FileManager.default.createDirectory(
                at: exoHomeURL, withIntermediateDirectories: true
@@ -216,7 +217,6 @@ final class ExoProcessController: ObservableObject {
    private func makeEnvironment(for runtimeURL: URL) -> [String: String] {
        var environment = ProcessInfo.processInfo.environment
        environment["EXO_RUNTIME_DIR"] = runtimeURL.path
-        environment["EXO_LIBP2P_NAMESPACE"] = computeNamespace()
        if !hfToken.isEmpty {
            environment["HF_TOKEN"] = hfToken
        }
@@ -0,0 +1,7 @@
+# Canary benchmark manifest
+#
+# Lists the suite files to include. Each file defines benchmarks
+# with shared constraints, topology, and default args.
+include = [
+    "single-m3-ultra.toml",
+]
@@ -0,0 +1,189 @@
+# Single-node M3 Ultra benchmarks
+#
+# Shared constraints applied to ALL benchmarks in this file.
+constraints = [
+    "All(MacOsBuild(=25D125))",
+    "Hosts(=1)",
+    "All(Chip(m3_ultra))",
+    "All(GpuCores(=80))",
+]
+
+[topology]
+type = "none"
+
+# Default args merged into each benchmark's args (benchmark-level args win).
+[defaults]
+pp = [512, 2048, 8192, 16384]
+tg = 128
+
+[[benchmark]]
+model = "mlx-community/Meta-Llama-3.1-70B-Instruct-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/gpt-oss-120b-MXFP4-Q8"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.7-Flash-8bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Coder-Next-6bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-30B-A3B-8bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-0.6B-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-0.6B-8bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Llama-3.2-1B-Instruct-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Llama-3.2-3B-Instruct-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Llama-3.2-3B-Instruct-8bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Meta-Llama-3.1-8B-Instruct-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Meta-Llama-3.1-8B-Instruct-8bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Meta-Llama-3.1-8B-Instruct-bf16"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/gpt-oss-20b-MXFP4-Q8"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-30B-A3B-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.7-Flash-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.7-Flash-5bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.7-Flash-6bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Llama-3.3-70B-Instruct-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Coder-Next-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Coder-Next-5bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Coder-Next-8bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Next-80B-A3B-Instruct-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Next-80B-A3B-Instruct-8bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Next-80B-A3B-Thinking-4bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Next-80B-A3B-Thinking-8bit"
+extra_constraints = ["All(Memory(>=96GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Llama-3.3-70B-Instruct-8bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/llama-3.3-70b-instruct-fp16"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.5-Air-8bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.5-Air-bf16"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.7-4bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/MiniMax-M2.1-3bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/MiniMax-M2.1-8bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-235B-A22B-Instruct-2507-4bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Coder-Next-bf16"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Step-3.5-Flash-4bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Step-3.5-Flash-6bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Step-3.5-Flash-8Bit"
+extra_constraints = ["All(Memory(>=256GiB))"]
+
+[[benchmark]]
+model = "mlx-community/DeepSeek-V3.1-4bit"
+extra_constraints = ["All(Memory(>=512GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.7-6bit"
+extra_constraints = ["All(Memory(>=512GiB))"]
+
+[[benchmark]]
+model = "mlx-community/GLM-4.7-8bit-gs32"
+extra_constraints = ["All(Memory(>=512GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-235B-A22B-Instruct-2507-8bit"
+extra_constraints = ["All(Memory(>=512GiB))"]
+
+[[benchmark]]
+model = "mlx-community/Qwen3-Coder-480B-A35B-Instruct-4bit"
+extra_constraints = ["All(Memory(>=512GiB))"]
@@ -1 +0,0 @@
-collect_ignore = ["tests/start_distributed_test.py"]
@@ -1,58 +0,0 @@
-# Stage 1: Build the dashboard
-FROM node:22-slim AS dashboard
-WORKDIR /app/dashboard
-COPY dashboard/package.json dashboard/package-lock.json ./
-RUN npm ci
-COPY dashboard/ .
-RUN npm run build
-
-# Stage 2: Build and run exo
-FROM python:3.13-slim
-
-# Install system dependencies
-# libblas-dev/liblapack-dev/liblapacke-dev are required by MLX CPU backend on Linux
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    pkg-config \
-    libssl-dev \
-    libblas-dev \
-    liblapack-dev \
-    liblapacke-dev \
-    curl \
-    protobuf-compiler \
-    iptables \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install Rust nightly
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
-ENV PATH="/root/.cargo/bin:${PATH}"
-
-# Wrap g++ with -fpermissive to fix MLX CPU JIT compilation with GCC 14
-# (GCC 14 treats _Float128/_Float32/_Float64 as built-in types, conflicting with MLX-generated code)
-# Must be done BEFORE uv sync so any source builds also get the fix
-RUN mv /usr/bin/g++ /usr/bin/g++.real && \
-    printf '#!/bin/sh\nexec /usr/bin/g++.real -fpermissive "$@"\n' > /usr/bin/g++ && \
-    chmod +x /usr/bin/g++
-
-# Install uv
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
-
-WORKDIR /app
-
-# Copy dependency files first for better layer caching
-COPY pyproject.toml Cargo.toml uv.lock README.md ./
-COPY rust/ ./rust/
-COPY bench/pyproject.toml ./bench/pyproject.toml
-
-# Copy source and resources
-COPY src/ ./src/
-COPY resources/ ./resources/
-
-# Copy built dashboard from stage 1
-COPY --from=dashboard /app/dashboard/build ./dashboard/build/
-
-# Install Python deps and build Rust bindings, then clean up build artifacts
-# to keep the layer small (Rust target/ and cargo registry can be 1-2 GB)
-RUN uv sync && rm -rf /app/rust/target /root/.cargo/registry /root/.cargo/git
-
-CMD [".venv/bin/exo", "-v"]
@@ -1,195 +0,0 @@
-"""Shared E2E test infrastructure for exo cluster tests."""
-
-import asyncio
-import json
-import os
-import sys
-from pathlib import Path
-from urllib.error import URLError
-from urllib.request import Request, urlopen
-
-E2E_DIR = Path(__file__).parent.resolve()
-TIMEOUT = int(os.environ.get("E2E_TIMEOUT", "120"))
-
-
-class Cluster:
-    """Async wrapper around a docker compose exo cluster."""
-
-    def __init__(self, name: str, overrides: list[str] | None = None):
-        self.name = name
-        self.project = f"e2e-{name}"
-        compose_files = [str(E2E_DIR / "docker-compose.yml")]
-        for path in overrides or []:
-            compose_files.append(str(E2E_DIR / path))
-        self._compose_base = [
-            "docker",
-            "compose",
-            "-p",
-            self.project,
-            *[arg for f in compose_files for arg in ("-f", f)],
-        ]
-
-    async def __aenter__(self):
-        return self
-
-    async def __aexit__(self, *exc):
-        await self.stop()
-
-    async def _run(self, *args: str, check: bool = True) -> str:
-        proc = await asyncio.create_subprocess_exec(
-            *self._compose_base,
-            *args,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.STDOUT,
-        )
-        stdout, _ = await proc.communicate()
-        output = stdout.decode()
-        if check and proc.returncode != 0:
-            print(output, file=sys.stderr)
-            raise RuntimeError(
-                f"docker compose {' '.join(args)} failed (rc={proc.returncode})"
-            )
-        return output
-
-    async def build(self):
-        # Skip build if the image was pre-built (e.g. in CI with buildx cache)
-        proc = await asyncio.create_subprocess_exec(
-            "docker",
-            "image",
-            "inspect",
-            "exo-e2e:latest",
-            stdout=asyncio.subprocess.DEVNULL,
-            stderr=asyncio.subprocess.DEVNULL,
-        )
-        await proc.wait()
-        if proc.returncode == 0:
-            print("  Using pre-built image (exo-e2e:latest)")
-            return
-        print("  Building images...")
-        await self._run("build", "--quiet")
-
-    async def start(self):
-        print("  Starting cluster...")
-        await self._run("up", "-d")
-
-    async def stop(self):
-        print("  Cleaning up...")
-        await self._run("down", "--timeout", "5", check=False)
-
-    async def logs(self) -> str:
-        return await self._run("logs", check=False)
-
-    async def exec(
-        self, service: str, *cmd: str, check: bool = True
-    ) -> tuple[int, str]:
-        """Run a command inside a running container. Returns (returncode, output)."""
-        proc = await asyncio.create_subprocess_exec(
-            *self._compose_base,
-            "exec",
-            "-T",
-            service,
-            *cmd,
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.STDOUT,
-        )
-        stdout, _ = await proc.communicate()
-        output = stdout.decode()
-        if check and proc.returncode != 0:
-            raise RuntimeError(
-                f"exec {' '.join(cmd)} in {service} failed (rc={proc.returncode})"
-            )
-        return proc.returncode, output
-
-    async def wait_for(self, description: str, check_fn, timeout: int = TIMEOUT):
-        """Poll check_fn every 2s until it returns True or timeout expires."""
-        print(f"  Waiting for {description}...")
-        deadline = asyncio.get_event_loop().time() + timeout
-        while asyncio.get_event_loop().time() < deadline:
-            if await check_fn():
-                print(f"  {description}")
-                return
-            await asyncio.sleep(2)
-        output = await self.logs()
-        print(f"--- cluster logs ---\n{output}\n---", file=sys.stderr)
-        raise TimeoutError(f"Timed out waiting for {description}")
-
-    async def assert_healthy(self):
-        """Verify the cluster formed correctly: nodes started, discovered each other, elected a master, API responds."""
-
-        async def both_nodes_started():
-            log = await self.logs()
-            return log.count("Starting node") >= 2
-
-        async def nodes_discovered():
-            log = await self.logs()
-            return log.count("ConnectionMessageType.Connected") >= 2
-
-        async def master_elected():
-            log = await self.logs()
-            return "demoting self" in log
-
-        async def api_responding():
-            try:
-                with urlopen("http://localhost:52415/v1/models", timeout=3) as resp:
-                    return resp.status == 200
-            except (URLError, OSError):
-                return False
-
-        await self.wait_for("Both nodes started", both_nodes_started)
-        await self.wait_for("Nodes discovered each other", nodes_discovered)
-        await self.wait_for("Master election resolved", master_elected)
-        await self.wait_for("API responding", api_responding)
-
-    async def _api(
-        self, method: str, path: str, body: dict | None = None, timeout: int = 30
-    ) -> dict:
-        """Make an API request to the cluster. Returns parsed JSON."""
-        url = f"http://localhost:52415{path}"
-        data = json.dumps(body).encode() if body else None
-        req = Request(
-            url, data=data, headers={"Content-Type": "application/json"}, method=method
-        )
-        loop = asyncio.get_event_loop()
-        resp_bytes = await loop.run_in_executor(
-            None, lambda: urlopen(req, timeout=timeout).read()
-        )
-        return json.loads(resp_bytes)
-
-    async def place_model(self, model: str, timeout: int = 600):
-        """Place a model instance on the cluster (triggers download) and wait until it's ready."""
-        await self._api("POST", "/place_instance", {"model_id": model})
-
-        async def model_ready():
-            try:
-                resp = await self._api("GET", "/v1/models")
-                return any(m.get("id") == model for m in resp.get("data", []))
-            except Exception:
-                return False
-
-        await self.wait_for(f"Model {model} ready", model_ready, timeout=timeout)
-
-    async def chat(
-        self, model: str, messages: list[dict], timeout: int = 600, **kwargs
-    ) -> dict:
-        """Send a chat completion request. Retries until model is downloaded and inference completes."""
-        body = json.dumps({"model": model, "messages": messages, **kwargs}).encode()
-        deadline = asyncio.get_event_loop().time() + timeout
-        last_error = None
-
-        while asyncio.get_event_loop().time() < deadline:
-            try:
-                req = Request(
-                    "http://localhost:52415/v1/chat/completions",
-                    data=body,
-                    headers={"Content-Type": "application/json"},
-                )
-                loop = asyncio.get_event_loop()
-                resp_bytes = await loop.run_in_executor(
-                    None, lambda r=req: urlopen(r, timeout=300).read()
-                )
-                return json.loads(resp_bytes)
-            except Exception as e:
-                last_error = e
-                await asyncio.sleep(5)
-
-        raise TimeoutError(f"Chat request failed after {timeout}s: {last_error}")
@@ -1,20 +0,0 @@
-services:
-  exo-node-1:
-    image: exo-e2e:latest
-    build:
-      context: ..
-      dockerfile: e2e/Dockerfile
-    environment:
-      - EXO_LIBP2P_NAMESPACE=docker-e2e
-    command: [".venv/bin/exo", "-v"]
-    ports:
-      - "52415:52415"
-
-  exo-node-2:
-    image: exo-e2e:latest
-    build:
-      context: ..
-      dockerfile: e2e/Dockerfile
-    environment:
-      - EXO_LIBP2P_NAMESPACE=docker-e2e
-    command: [".venv/bin/exo", "-v"]
@@ -1,83 +0,0 @@
-#!/usr/bin/env python3
-"""Discovers and runs all E2E tests in e2e/test_*.py.
-
-Tests with '# slow' on the first line of their docstring are skipped
-unless --slow is passed or E2E_SLOW=1 is set.
-"""
-
-import os
-import subprocess
-import sys
-from pathlib import Path
-
-E2E_DIR = Path(__file__).parent.resolve()
-
-
-def is_slow(test_file: Path) -> bool:
-    """Check if the test file is marked as slow (has '# slow' in first 3 lines)."""
-    with open(test_file) as f:
-        for line in f:
-            if line.strip().startswith("#"):
-                continue
-            if line.strip().startswith('"""') or line.strip().startswith("'''"):
-                # Read into the docstring
-                for doc_line in f:
-                    if "slow" in doc_line.lower() and doc_line.strip().startswith(
-                        "slow"
-                    ):
-                        return True
-                    if '"""' in doc_line or "'''" in doc_line:
-                        break
-            break
-    return False
-
-
-def main():
-    run_slow = "--slow" in sys.argv or os.environ.get("E2E_SLOW") == "1"
-    if "--update-snapshots" in sys.argv:
-        os.environ["UPDATE_SNAPSHOTS"] = "1"
-    test_files = sorted(E2E_DIR.glob("test_*.py"))
-    if not test_files:
-        print("No test files found")
-        sys.exit(1)
-
-    passed = 0
-    failed = 0
-    skipped = 0
-    failures = []
-
-    for test_file in test_files:
-        name = test_file.stem
-        if is_slow(test_file) and not run_slow:
-            print(f"=== {name} === SKIPPED (slow, use --slow to run)")
-            skipped += 1
-            continue
-
-        print(f"=== {name} ===")
-        result = subprocess.run([sys.executable, str(test_file)])
-        if result.returncode == 0:
-            passed += 1
-        else:
-            # Retry once — Docker networking (mDNS) can be slow on first boot
-            print(f"\n=== {name} === RETRYING (attempt 2/2)")
-            result = subprocess.run([sys.executable, str(test_file)])
-            if result.returncode == 0:
-                passed += 1
-            else:
-                failed += 1
-                failures.append(name)
-        print()
-
-    total = passed + failed + skipped
-    print("================================")
-    print(
-        f"{passed}/{total} tests passed" + (f", {skipped} skipped" if skipped else "")
-    )
-
-    if failed:
-        print(f"Failed: {' '.join(failures)}")
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()
@@ -1,78 +0,0 @@
-"""Snapshot testing infrastructure for E2E tests.
-
-Provides deterministic regression testing by comparing inference output
-against committed baseline snapshots. Tests FAIL if no baseline exists —
-baselines must be explicitly generated and committed.
-
-Generate baselines: UPDATE_SNAPSHOTS=1 python3 e2e/run_all.py --slow
-Update after intentional changes: UPDATE_SNAPSHOTS=1 python3 e2e/run_all.py --slow
-
-Snapshots are stored per-architecture (e.g. snapshots/x86_64/, snapshots/arm64/)
-since floating-point results differ between CPU architectures.
-"""
-
-import difflib
-import json
-import os
-import platform
-from pathlib import Path
-
-ARCH = platform.machine()
-SNAPSHOTS_DIR = Path(__file__).parent / "snapshots" / ARCH
-
-
-def assert_snapshot(
-    name: str,
-    content: str,
-    metadata: dict,
-) -> None:
-    """Compare content against a saved snapshot, or create one if missing.
-
-    Args:
-        name: Snapshot identifier (used as filename: snapshots/{arch}/{name}.json).
-        content: The actual inference output to compare.
-        metadata: Additional context stored alongside content (model, seed, etc.).
-                  Not used for comparison -- purely documentary.
-
-    Raises:
-        AssertionError: If content doesn't match the saved snapshot.
-
-    Environment:
-        UPDATE_SNAPSHOTS=1: Overwrite existing snapshot with actual content.
-    """
-    snapshot_file = SNAPSHOTS_DIR / f"{name}.json"
-    update = os.environ.get("UPDATE_SNAPSHOTS") == "1"
-
-    if update:
-        # Explicitly regenerate snapshot
-        SNAPSHOTS_DIR.mkdir(parents=True, exist_ok=True)
-        snapshot_data = {**metadata, "arch": ARCH, "content": content}
-        snapshot_file.write_text(json.dumps(snapshot_data, indent=2) + "\n")
-        print(f"  Updated snapshot: {ARCH}/{snapshot_file.name}")
-    elif not snapshot_file.exists():
-        raise AssertionError(
-            f"No baseline snapshot for '{name}' on {ARCH}.\n"
-            f"Expected file: {snapshot_file}\n\n"
-            f"Generate baselines with: UPDATE_SNAPSHOTS=1 python3 e2e/run_all.py --slow"
-        )
-    else:
-        snapshot = json.loads(snapshot_file.read_text())
-        expected = snapshot["content"]
-        if content != expected:
-            diff = "\n".join(
-                difflib.unified_diff(
-                    expected.splitlines(),
-                    content.splitlines(),
-                    fromfile=f"expected ({snapshot_file.relative_to(SNAPSHOTS_DIR.parent.parent)})",
-                    tofile="actual",
-                    lineterm="",
-                )
-            )
-            raise AssertionError(
-                f"Snapshot mismatch for '{name}' on {ARCH}!\n\n"
-                f"{diff}\n\n"
-                f"Expected: {expected!r}\n"
-                f"Actual:   {content!r}\n\n"
-                f"To update: UPDATE_SNAPSHOTS=1 python3 e2e/run_all.py --slow"
-            )
-        print(f"  Output matches snapshot ({ARCH}/{snapshot_file.name})")
@@ -1,22 +0,0 @@
-"""Test: Basic cluster formation.
-
-Verifies two nodes discover each other, elect a master, and the API responds.
-"""
-
-import asyncio
-import sys
-
-sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent))
-from conftest import Cluster
-
-
-async def main():
-    async with Cluster("cluster_formation") as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-        print("PASSED: cluster_formation")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,61 +0,0 @@
-"""Test: Deterministic inference output (snapshot test).
-
-Sends a chat completion request with a fixed seed,
-then verifies the output matches a known-good snapshot. This ensures
-inference produces consistent results across runs.
-
-Uses MLX CPU backend in Docker on x86 Linux.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from snapshot import assert_snapshot
-
-from conftest import Cluster
-
-MODEL = "mlx-community/Qwen3-0.6B-4bit"
-SEED = 42
-PROMPT = "What is 2+2? Reply with just the number."
-MAX_TOKENS = 32
-
-
-async def main():
-    async with Cluster("inference_snapshot") as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-
-        print(f"  Launching model {MODEL}...")
-        await cluster.place_model(MODEL)
-
-        print(f"  Sending chat completion (seed={SEED})...")
-        resp = await cluster.chat(
-            model=MODEL,
-            messages=[{"role": "user", "content": PROMPT}],
-            seed=SEED,
-            temperature=0,
-            max_tokens=MAX_TOKENS,
-        )
-
-        content = resp["choices"][0]["message"]["content"]
-        print(f"  Response: {content!r}")
-
-        assert_snapshot(
-            name="inference_snapshot",
-            content=content,
-            metadata={
-                "model": MODEL,
-                "seed": SEED,
-                "prompt": PROMPT,
-                "max_tokens": MAX_TOKENS,
-            },
-        )
-
-        print("PASSED: inference_snapshot")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,47 +0,0 @@
-"""Test: Cluster works without internet access.
-
-Verifies exo functions correctly when containers can talk to each other
-but cannot reach the internet. Uses iptables to block all outbound traffic
-except private subnets and multicast (for mDNS discovery).
-"""
-
-import asyncio
-import sys
-
-sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent))
-from conftest import Cluster
-
-
-async def main():
-    async with Cluster(
-        "no_internet",
-        overrides=["tests/no_internet/docker-compose.override.yml"],
-    ) as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-
-        # Verify internet is actually blocked from inside the containers
-        for node in ["exo-node-1", "exo-node-2"]:
-            rc, _ = await cluster.exec(
-                node,
-                "curl",
-                "-sf",
-                "--max-time",
-                "3",
-                "https://huggingface.co",
-                check=False,
-            )
-            assert rc != 0, f"{node} should not be able to reach the internet"
-            print(f"  {node}: internet correctly blocked")
-
-        # Verify exo detected no internet connectivity
-        log = await cluster.logs()
-        assert "Internet connectivity: False" in log, "exo should detect no internet"
-        print("  exo correctly detected no internet connectivity")
-
-        print("PASSED: no_internet")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,65 +0,0 @@
-"""Test: Runner chaos — abrupt runner death detection.
-slow
-
-Sends a chat completion with the EXO_RUNNER_MUST_DIE trigger, which causes
-the runner process to call os._exit(1) (simulating an OOM kill). Verifies that
-the RunnerSupervisor health check detects the death and the system doesn't hang.
-
-Requires a machine that can run MLX inference at reasonable speed (Apple Silicon).
-Run with: python3 e2e/run_all.py --slow  or  E2E_SLOW=1 python3 e2e/run_all.py
-"""
-
-import asyncio
-import contextlib
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from conftest import Cluster
-
-MODEL = "mlx-community/Qwen3-0.6B-4bit"
-
-
-async def main():
-    async with Cluster("runner_chaos") as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-
-        # Place the model so a runner is loaded and ready
-        print(f"  Launching model {MODEL}...")
-        await cluster.place_model(MODEL)
-
-        # Send a chat request with the die trigger.
-        # The runner will call os._exit(1) mid-inference, simulating OOM kill.
-        # The chat request itself will fail — that's expected.
-        print("  Sending EXO_RUNNER_MUST_DIE trigger...")
-        with contextlib.suppress(Exception):
-            await cluster.chat(
-                model=MODEL,
-                messages=[{"role": "user", "content": "EXO RUNNER MUST DIE"}],
-                timeout=60,
-            )
-
-        # Wait for the health check to detect the death and emit RunnerFailed
-        async def health_check_detected():
-            log = await cluster.logs()
-            return "runner process died unexpectedly" in log
-
-        await cluster.wait_for(
-            "Health check detected runner death",
-            health_check_detected,
-            timeout=30,
-        )
-
-        # Verify RunnerFailed was emitted (visible in logs)
-        log = await cluster.logs()
-        assert "runner process died unexpectedly" in log, (
-            f"Expected health check to detect runner death but it didn't.\nLogs:\n{log}"
-        )
-
-        print("PASSED: runner_chaos")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,60 +0,0 @@
-"""Test: Code generation snapshot.
-slow
-
-Verifies deterministic output for a code generation prompt.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from snapshot import assert_snapshot
-
-from conftest import Cluster
-
-MODEL = "mlx-community/Qwen3-0.6B-4bit"
-SEED = 42
-PROMPT = (
-    "Write a Python function to reverse a string. Only output the code, no explanation."
-)
-MAX_TOKENS = 64
-
-
-async def main():
-    async with Cluster("snapshot_code_gen") as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-
-        print(f"  Launching model {MODEL}...")
-        await cluster.place_model(MODEL)
-
-        print(f"  Sending chat completion (seed={SEED})...")
-        resp = await cluster.chat(
-            model=MODEL,
-            messages=[{"role": "user", "content": PROMPT}],
-            seed=SEED,
-            temperature=0,
-            max_tokens=MAX_TOKENS,
-        )
-
-        content = resp["choices"][0]["message"]["content"]
-        print(f"  Response: {content!r}")
-
-        assert_snapshot(
-            name="snapshot_code_gen",
-            content=content,
-            metadata={
-                "model": MODEL,
-                "seed": SEED,
-                "prompt": PROMPT,
-                "max_tokens": MAX_TOKENS,
-            },
-        )
-
-        print("PASSED: snapshot_code_gen")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,65 +0,0 @@
-"""Test: Edge case snapshots.
-slow
-
-Verifies deterministic output for edge-case prompts: single word input,
-special characters, and unicode.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from snapshot import assert_snapshot
-
-from conftest import Cluster
-
-MODEL = "mlx-community/Qwen3-0.6B-4bit"
-SEED = 42
-MAX_TOKENS = 32
-
-CASES = [
-    ("edge_single_word", "Hi"),
-    ("edge_special_chars", "What does 2 * (3 + 4) / 7 - 1 equal? Use <math> tags."),
-    ("edge_unicode", "Translate 'hello' to Japanese, Chinese, and Korean."),
-]
-
-
-async def main():
-    async with Cluster("snapshot_edge") as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-
-        print(f"  Launching model {MODEL}...")
-        await cluster.place_model(MODEL)
-
-        for snapshot_name, prompt in CASES:
-            print(f"  [{snapshot_name}] Sending: {prompt!r}")
-            resp = await cluster.chat(
-                model=MODEL,
-                messages=[{"role": "user", "content": prompt}],
-                seed=SEED,
-                temperature=0,
-                max_tokens=MAX_TOKENS,
-            )
-
-            content = resp["choices"][0]["message"]["content"]
-            print(f"  [{snapshot_name}] Response: {content!r}")
-
-            assert_snapshot(
-                name=snapshot_name,
-                content=content,
-                metadata={
-                    "model": MODEL,
-                    "seed": SEED,
-                    "prompt": prompt,
-                    "max_tokens": MAX_TOKENS,
-                },
-            )
-
-        print("PASSED: snapshot_edge")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,58 +0,0 @@
-"""Test: Longer output snapshot.
-slow
-
-Verifies deterministic output with a higher max_tokens (128).
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from snapshot import assert_snapshot
-
-from conftest import Cluster
-
-MODEL = "mlx-community/Qwen3-0.6B-4bit"
-SEED = 42
-PROMPT = "Explain how a binary search algorithm works."
-MAX_TOKENS = 128
-
-
-async def main():
-    async with Cluster("snapshot_long_output") as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-
-        print(f"  Launching model {MODEL}...")
-        await cluster.place_model(MODEL)
-
-        print(f"  Sending chat completion (seed={SEED}, max_tokens={MAX_TOKENS})...")
-        resp = await cluster.chat(
-            model=MODEL,
-            messages=[{"role": "user", "content": PROMPT}],
-            seed=SEED,
-            temperature=0,
-            max_tokens=MAX_TOKENS,
-        )
-
-        content = resp["choices"][0]["message"]["content"]
-        print(f"  Response: {content!r}")
-
-        assert_snapshot(
-            name="snapshot_long_output",
-            content=content,
-            metadata={
-                "model": MODEL,
-                "seed": SEED,
-                "prompt": PROMPT,
-                "max_tokens": MAX_TOKENS,
-            },
-        )
-
-        print("PASSED: snapshot_long_output")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,73 +0,0 @@
-"""Test: Multi-model snapshot tests.
-slow
-
-Verifies deterministic output across different model architectures to catch
-model-specific regressions. Each model uses its own snapshot file.
-Run with: python3 e2e/run_all.py --slow  or  E2E_SLOW=1 python3 e2e/run_all.py
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from snapshot import assert_snapshot
-
-from conftest import Cluster
-
-SEED = 42
-PROMPT = "What is the capital of France?"
-MAX_TOKENS = 32
-
-MODELS = [
-    "mlx-community/SmolLM2-135M-Instruct",
-    "mlx-community/Llama-3.2-1B-Instruct-4bit",
-    "mlx-community/gemma-2-2b-it-4bit",
-]
-
-
-async def main():
-    async with Cluster("snapshot_multi_model") as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-
-        for model in MODELS:
-            short_name = (
-                model.split("/")[-1].lower().replace("-", "_").replace(".", "_")
-            )
-            snapshot_name = f"snapshot_multi_{short_name}"
-
-            print(f"  Launching model {model}...")
-            await cluster.place_model(model)
-
-            print(f"  Sending chat completion (seed={SEED})...")
-            resp = await cluster.chat(
-                model=model,
-                messages=[{"role": "user", "content": PROMPT}],
-                seed=SEED,
-                temperature=0,
-                max_tokens=MAX_TOKENS,
-            )
-
-            content = resp["choices"][0]["message"]["content"]
-            print(f"  [{short_name}] Response: {content!r}")
-
-            assert_snapshot(
-                name=snapshot_name,
-                content=content,
-                metadata={
-                    "model": model,
-                    "seed": SEED,
-                    "prompt": PROMPT,
-                    "max_tokens": MAX_TOKENS,
-                },
-            )
-
-            print(f"  [{short_name}] PASSED")
-
-        print("PASSED: snapshot_multi_model")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,58 +0,0 @@
-"""Test: Reasoning/math snapshot.
-slow
-
-Verifies deterministic output for a simple reasoning prompt.
-"""
-
-import asyncio
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).parent))
-from snapshot import assert_snapshot
-
-from conftest import Cluster
-
-MODEL = "mlx-community/Qwen3-0.6B-4bit"
-SEED = 42
-PROMPT = "If I have 3 apples and give away 1, how many do I have? Think step by step."
-MAX_TOKENS = 64
-
-
-async def main():
-    async with Cluster("snapshot_reasoning") as cluster:
-        await cluster.build()
-        await cluster.start()
-        await cluster.assert_healthy()
-
-        print(f"  Launching model {MODEL}...")
-        await cluster.place_model(MODEL)
-
-        print(f"  Sending chat completion (seed={SEED})...")
-        resp = await cluster.chat(
-            model=MODEL,
-            messages=[{"role": "user", "content": PROMPT}],
-            seed=SEED,
-            temperature=0,
-            max_tokens=MAX_TOKENS,
-        )
-
-        content = resp["choices"][0]["message"]["content"]
-        print(f"  Response: {content!r}")
-
-        assert_snapshot(
-            name="snapshot_reasoning",
-            content=content,
-            metadata={
-                "model": MODEL,
-                "seed": SEED,
-                "prompt": PROMPT,
-                "max_tokens": MAX_TOKENS,
-            },
-        )
-
-        print("PASSED: snapshot_reasoning")
-
-
-if __name__ == "__main__":
-    asyncio.run(main())
@@ -1,32 +0,0 @@
-# Block all outbound internet traffic using iptables while preserving:
-#   - Multicast (224.0.0.0/4) for mDNS peer discovery
-#   - Private subnets (10/8, 172.16/12, 192.168/16) for inter-container communication
-#   - Loopback (127/8)
-# Requires NET_ADMIN capability for iptables.
-services:
-  exo-node-1:
-    cap_add:
-      - NET_ADMIN
-    entrypoint: ["/bin/sh", "-c"]
-    command:
-      - |
-        iptables -A OUTPUT -d 127.0.0.0/8 -j ACCEPT
-        iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT
-        iptables -A OUTPUT -d 172.16.0.0/12 -j ACCEPT
-        iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT
-        iptables -A OUTPUT -d 224.0.0.0/4 -j ACCEPT
-        iptables -A OUTPUT -j REJECT
-        exec .venv/bin/exo -v
-  exo-node-2:
-    cap_add:
-      - NET_ADMIN
-    entrypoint: ["/bin/sh", "-c"]
-    command:
-      - |
-        iptables -A OUTPUT -d 127.0.0.0/8 -j ACCEPT
-        iptables -A OUTPUT -d 10.0.0.0/8 -j ACCEPT
-        iptables -A OUTPUT -d 172.16.0.0/12 -j ACCEPT
-        iptables -A OUTPUT -d 192.168.0.0/16 -j ACCEPT
-        iptables -A OUTPUT -d 224.0.0.0/4 -j ACCEPT
-        iptables -A OUTPUT -j REJECT
-        exec .venv/bin/exo -v
@@ -1,12 +0,0 @@
-model_id = "mlx-community/SmolLM2-135M-Instruct"
-n_layers = 30
-hidden_size = 576
-supports_tensor = true
-tasks = ["TextGeneration"]
-family = "llama"
-quantization = "bf16"
-base_model = "SmolLM2 135M"
-capabilities = ["text"]
-
-[storage_size]
-in_bytes = 269060381
@@ -1,12 +0,0 @@
-model_id = "mlx-community/gemma-2-2b-it-4bit"
-n_layers = 26
-hidden_size = 2304
-supports_tensor = false
-tasks = ["TextGeneration"]
-family = "gemma2"
-quantization = "4bit"
-base_model = "Gemma 2 2B"
-capabilities = ["text"]
-
-[storage_size]
-in_bytes = 1492755242
@@ -1,2 +0,0 @@
-# we can manually exclude false-positive lint errors for dual packages (if in dependencies)
-#allowed-duplicate-crates = ["hashbrown"]
@@ -25,44 +25,26 @@ workspace = true
 networking = { workspace = true }

 # interop
-pyo3 = { version = "0.27.1", features = [
-    # "abi3-py311", # tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.11
-    "nightly", # enables better-supported GIL integration
-    "experimental-async", # async support in #[pyfunction] & #[pymethods]
-    #"experimental-inspect", # inspection of generated binary => easier to automate type-hint generation
-    #"py-clone", # adding Clone-ing of `Py<T>` without GIL (may cause panics - remove if panics happen)
-    "multiple-pymethods", # allows multiple #[pymethods] sections per class
+pyo3 = { version = "0.27.2", features = [
+    "abi3-py313", # tells pyo3 (and maturin) to build using the stable ABI with minimum Python version 3.13
+    # "nightly", # enables better-supported GIL integration
+    "experimental-async" # async support in #[pyfunction] & #[pymethods]
+    # "experimental-inspect", # inspection of generated binary => easier to automate type-hint generation
+    # "py-clone", # adding Clone-ing of `Py<T>` without GIL (may cause panics - remove if panics happen)
+    # "multiple-pymethods", # allows multiple #[pymethods] sections per class

    # integrations with other libraries
-    "arc_lock", "bigdecimal", "either", "hashbrown", "indexmap", "num-bigint", "num-complex", "num-rational",
-    "ordered-float", "rust_decimal", "smallvec",
+    # "arc_lock", "bigdecimal", "either", "hashbrown", "indexmap", "num-bigint", "num-complex", "num-rational",
+    # "ordered-float", "rust_decimal", "smallvec",
    # "anyhow", "chrono", "chrono-local", "chrono-tz", "eyre", "jiff-02", "lock_api", "parking-lot", "time",  "serde",
 ] }
 pyo3-stub-gen = { version = "0.17.2" }
 pyo3-async-runtimes = { version = "0.27.0", features = ["attributes", "tokio-runtime", "testing"] }
 pyo3-log = "0.13.2"

-# macro dependencies
-extend = { workspace = true }
-delegate = { workspace = true }
-impl-trait-for-tuples = { workspace = true }
-derive_more = { workspace = true }
-pin-project = { workspace = true }
-
 # async runtime
 tokio = { workspace = true, features = ["full", "tracing"] }
-futures = { workspace = true }
-
-# utility dependencies
-once_cell = "1.21.3"
-thread_local = "1.1.9"
-util = { workspace = true }
-thiserror = { workspace = true }
-#internment = { workspace = true }
-#recursion = { workspace = true }
-#generativity = { workspace = true }
-#itertools = { workspace = true }
-
+futures-lite = "2.6.1"

 # Tracing
 #tracing = "0.1"
@@ -2,220 +2,39 @@
 # ruff: noqa: E501, F401

 import builtins
-import enum
 import typing

-@typing.final
-class AllQueuesFullError(builtins.Exception):
-    def __new__(cls, *args: typing.Any) -> AllQueuesFullError: ...
-    def __repr__(self) -> builtins.str: ...
-    def __str__(self) -> builtins.str: ...
-
-@typing.final
-class ConnectionUpdate:
-    @property
-    def update_type(self) -> ConnectionUpdateType:
-        r"""
-        Whether this is a connection or disconnection event
-        """
-    @property
-    def peer_id(self) -> PeerId:
-        r"""
-        Identity of the peer that we have connected to or disconnected from.
-        """
-    @property
-    def remote_ipv4(self) -> builtins.str:
-        r"""
-        Remote connection's IPv4 address.
-        """
-    @property
-    def remote_tcp_port(self) -> builtins.int:
-        r"""
-        Remote connection's TCP port.
-        """
-
@typing.final
 class Keypair:
-    r"""
-    Identity keypair of a node.
-    """
    @staticmethod
-    def generate_ed25519() -> Keypair:
+    def generate() -> Keypair:
        r"""
-        Generate a new Ed25519 keypair.
-        """
-    @staticmethod
-    def generate_ecdsa() -> Keypair:
-        r"""
-        Generate a new ECDSA keypair.
-        """
-    @staticmethod
-    def generate_secp256k1() -> Keypair:
-        r"""
-        Generate a new Secp256k1 keypair.
+        Generate a new ed25519 keypair
        """
    @staticmethod
    def from_protobuf_encoding(bytes: bytes) -> Keypair:
        r"""
        Decode a private key from a protobuf structure and parse it as a `Keypair`.
        """
-    @staticmethod
-    def rsa_from_pkcs8(bytes: bytes) -> Keypair:
-        r"""
-        Decode an keypair from a DER-encoded secret key in PKCS#8 `PrivateKeyInfo`
-        format (i.e. unencrypted) as defined in [RFC5208].
-        
-        [RFC5208]: https://tools.ietf.org/html/rfc5208#section-5
-        """
-    @staticmethod
-    def secp256k1_from_der(bytes: bytes) -> Keypair:
-        r"""
-        Decode a keypair from a DER-encoded Secp256k1 secret key in an `ECPrivateKey`
-        structure as defined in [RFC5915].
-        
-        [RFC5915]: https://tools.ietf.org/html/rfc5915
-        """
-    @staticmethod
-    def ed25519_from_bytes(bytes: bytes) -> Keypair: ...
    def to_protobuf_encoding(self) -> bytes:
        r"""
-        Encode a private key as protobuf structure.
-        """
-    def to_peer_id(self) -> PeerId:
-        r"""
-        Convert the `Keypair` into the corresponding `PeerId`.
+        Encode a private key to a protobuf structure.
        """
+    def to_string(self) -> builtins.str: ...

@typing.final
-class Multiaddr:
-    r"""
-    Representation of a Multiaddr.
-    """
+class PyPeer:
    @staticmethod
-    def empty() -> Multiaddr:
-        r"""
-        Create a new, empty multiaddress.
-        """
-    @staticmethod
-    def with_capacity(n: builtins.int) -> Multiaddr:
-        r"""
-        Create a new, empty multiaddress with the given capacity.
-        """
-    @staticmethod
-    def from_bytes(bytes: bytes) -> Multiaddr:
-        r"""
-        Parse a `Multiaddr` value from its byte slice representation.
-        """
-    @staticmethod
-    def from_string(string: builtins.str) -> Multiaddr:
-        r"""
-        Parse a `Multiaddr` value from its string representation.
-        """
-    def len(self) -> builtins.int:
-        r"""
-        Return the length in bytes of this multiaddress.
-        """
-    def is_empty(self) -> builtins.bool:
-        r"""
-        Returns true if the length of this multiaddress is 0.
-        """
-    def to_bytes(self) -> bytes:
-        r"""
-        Return a copy of this [`Multiaddr`]'s byte representation.
-        """
-    def to_string(self) -> builtins.str:
-        r"""
-        Convert a Multiaddr to a string.
-        """
+    def new(kp: Keypair, namespace: builtins.str) -> PyPeer: ...
+    async def subscribe(self, topic: builtins.str) -> None: ...
+    async def unsubscribe(self, topic: builtins.str) -> None: ...
+    async def send(self, topic: builtins.str, payload: bytes) -> None: ...
+    async def run(self) -> None: ...
+    async def recv(self) -> PySwarmEvent: ...

@typing.final
-class NetworkingHandle:
-    def __new__(cls, identity: Keypair) -> NetworkingHandle: ...
-    async def connection_update_recv(self) -> ConnectionUpdate:
-        r"""
-        Receives the next `ConnectionUpdate` from networking.
-        """
-    async def connection_update_recv_many(self, limit: builtins.int) -> builtins.list[ConnectionUpdate]:
-        r"""
-        Receives at most `limit` `ConnectionUpdate`s from networking and returns them.
-        
-        For `limit = 0`, an empty collection of `ConnectionUpdate`s will be returned immediately.
-        For `limit > 0`, if there are no `ConnectionUpdate`s in the channel's queue this method
-        will sleep until a `ConnectionUpdate`s is sent.
-        """
-    async def gossipsub_subscribe(self, topic: builtins.str) -> builtins.bool:
-        r"""
-        Subscribe to a `GossipSub` topic.
-        
-        Returns `True` if the subscription worked. Returns `False` if we were already subscribed.
-        """
-    async def gossipsub_unsubscribe(self, topic: builtins.str) -> builtins.bool:
-        r"""
-        Unsubscribes from a `GossipSub` topic.
-        
-        Returns `True` if we were subscribed to this topic. Returns `False` if we were not subscribed.
-        """
-    async def gossipsub_publish(self, topic: builtins.str, data: bytes) -> None:
-        r"""
-        Publishes a message with multiple topics to the `GossipSub` network.
-        
-        If no peers are found that subscribe to this topic, throws `NoPeersSubscribedToTopicError` exception.
-        """
-    async def gossipsub_recv(self) -> tuple[builtins.str, bytes]:
-        r"""
-        Receives the next message from the `GossipSub` network.
-        """
-    async def gossipsub_recv_many(self, limit: builtins.int) -> builtins.list[tuple[builtins.str, bytes]]:
-        r"""
-        Receives at most `limit` messages from the `GossipSub` network and returns them.
-        
-        For `limit = 0`, an empty collection of messages will be returned immediately.
-        For `limit > 0`, if there are no messages in the channel's queue this method
-        will sleep until a message is sent.
-        """
-
-@typing.final
-class NoPeersSubscribedToTopicError(builtins.Exception):
-    def __new__(cls, *args: typing.Any) -> NoPeersSubscribedToTopicError: ...
-    def __repr__(self) -> builtins.str: ...
-    def __str__(self) -> builtins.str: ...
-
-@typing.final
-class PeerId:
-    r"""
-    Identifier of a peer of the network.
-    
-    The data is a `CIDv0` compatible multihash of the protobuf encoded public key of the peer
-    as specified in [specs/peer-ids](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md).
-    """
-    @staticmethod
-    def random() -> PeerId:
-        r"""
-        Generates a random peer ID from a cryptographically secure PRNG.
-        
-        This is useful for randomly walking on a DHT, or for testing purposes.
-        """
-    @staticmethod
-    def from_bytes(bytes: bytes) -> PeerId:
-        r"""
-        Parses a `PeerId` from bytes.
-        """
-    def to_bytes(self) -> bytes:
-        r"""
-        Returns a raw bytes representation of this `PeerId`.
-        """
-    def to_base58(self) -> builtins.str:
-        r"""
-        Returns a base-58 encoded string of this `PeerId`.
-        """
-    def __repr__(self) -> builtins.str: ...
-    def __str__(self) -> builtins.str: ...
-
-@typing.final
-class ConnectionUpdateType(enum.Enum):
-    r"""
-    Connection or disconnection event discriminant type.
-    """
-    Connected = ...
-    Disconnected = ...
+class PySwarmEvent:
+    def downcast_discovered(self) -> typing.Optional[builtins.str]: ...
+    def downcast_expired(self) -> typing.Optional[builtins.str]: ...
+    def downcast_message(self) -> typing.Optional[tuple[builtins.str, builtins.str, bytes]]: ...

@@ -1,8 +1,4 @@
-//! SEE: https://pyo3.rs/v0.26.0/async-await.html#detaching-from-the-interpreter-across-await
-//!
-
-use pin_project::pin_project;
-use pyo3::marker::Ungil;
+//! See: <https://pyo3.rs/v0.27.2/async-await.html#detaching-from-the-interpreter-across-await>
 use pyo3::prelude::*;
 use std::{
    future::Future,
@@ -10,31 +6,17 @@ use std::{
    task::{Context, Poll},
 };

-/// SEE: https://pyo3.rs/v0.26.0/async-await.html#detaching-from-the-interpreter-across-await
-#[pin_project]
-#[repr(transparent)]
-pub(crate) struct AllowThreads<F>(#[pin] F);
-
-impl<F> AllowThreads<F>
-where
-    Self: Future,
-{
-    pub fn new(f: F) -> Self {
-        Self(f)
-    }
-}
+pub struct AllowThreads<F>(pub(crate) F);

 impl<F> Future for AllowThreads<F>
 where
-    F: Future + Ungil,
-    F::Output: Ungil,
+    F: Future + Unpin + Send,
+    F::Output: Send,
 {
    type Output = F::Output;

-    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
        let waker = cx.waker();
-        Python::with_gil(|py| {
-            py.allow_threads(|| self.project().0.poll(&mut Context::from_waker(waker)))
-        })
+        Python::attach(|py| py.detach(|| pin!(&mut self.0).poll(&mut Context::from_waker(waker))))
    }
 }
@@ -1,240 +0,0 @@
-//! This module exists to hold examples of some pyo3 patterns that may be too complex to
-//! re-create from scratch, but too inhomogenous to create an abstraction/wrapper around.
-//!
-//! Pattern examples include:
-//!  - Async task handles: with GC-integrated cleanup
-//!  - Sync/async callbacks from python: with propper eventloop handling
-//!
-//! Mutability pattern: https://pyo3.rs/v0.26.0/async-await.html#send--static-constraint
-//!  - Store mutable fields in tokio's `Mutex<T>`
-//!  - For async code: take `&self` and `.lock().await`
-//!  - For sync code: take `&mut self` and `.get_mut()`
-
-use crate::ext::{PyResultExt as _, ResultExt as _, TokioRuntimeExt as _};
-use futures::FutureExt as _;
-use futures::future::BoxFuture;
-use pyo3::exceptions::PyRuntimeError;
-use pyo3::prelude::{PyModule, PyModuleMethods as _};
-use pyo3::{
-    Bound, Py, PyAny, PyErr, PyResult, PyTraverseError, PyVisit, Python, pyclass, pymethods,
-};
-use std::time::Duration;
-use tokio::sync::mpsc;
-use tokio::sync::mpsc::error::TryRecvError;
-
-fn needs_tokio_runtime() {
-    tokio::runtime::Handle::current();
-}
-
-type SyncCallback = Box<dyn Fn() + Send + Sync>;
-type AsyncCallback = Box<dyn Fn() -> BoxFuture<'static, ()> + Send + Sync>;
-
-enum AsyncTaskMessage {
-    SyncCallback(SyncCallback),
-    AsyncCallback(AsyncCallback),
-}
-
-async fn async_task(
-    sender: mpsc::UnboundedSender<()>,
-    mut receiver: mpsc::UnboundedReceiver<AsyncTaskMessage>,
-) {
-    log::info!("RUST: async task started");
-
-    // task state
-    let mut interval = tokio::time::interval(Duration::from_secs(1));
-
-    let mut sync_cbs: Vec<SyncCallback> = vec![];
-    let mut async_cbs: Vec<AsyncCallback> = vec![];
-
-    loop {
-        tokio::select! {
-            // handle incoming messages from task-handle
-            message = receiver.recv() => {
-                // handle closed channel by exiting
-                let Some(message) = message else {
-                    log::info!("RUST: channel closed");
-                    break;
-                };
-
-                // dispatch incoming event
-                match message {
-                    AsyncTaskMessage::SyncCallback(cb) => {
-                        sync_cbs.push(cb);
-                    }
-                    AsyncTaskMessage::AsyncCallback(cb) => {
-                        async_cbs.push(cb);
-                    }
-                }
-            }
-
-            // handle all other events
-            _ = interval.tick() => {
-                log::info!("RUST: async task tick");
-
-                // call back all sync callbacks
-                for cb in &sync_cbs {
-                    cb();
-                }
-
-                // call back all async callbacks
-                for cb in &async_cbs {
-                    cb().await;
-                }
-
-                // send event on unbounded channel
-                sender.send(()).expect("handle receiver cannot be closed/dropped");
-            }
-        }
-    }
-
-    log::info!("RUST: async task stopped");
-}
-
-// #[gen_stub_pyclass]
-#[pyclass(name = "AsyncTaskHandle")]
-#[derive(Debug)]
-struct PyAsyncTaskHandle {
-    sender: Option<mpsc::UnboundedSender<AsyncTaskMessage>>,
-    receiver: mpsc::UnboundedReceiver<()>,
-}
-
-#[allow(clippy::expect_used)]
-impl PyAsyncTaskHandle {
-    const fn sender(&self) -> &mpsc::UnboundedSender<AsyncTaskMessage> {
-        self.sender
-            .as_ref()
-            .expect("The sender should only be None after de-initialization.")
-    }
-
-    const fn sender_mut(&mut self) -> &mpsc::UnboundedSender<AsyncTaskMessage> {
-        self.sender
-            .as_mut()
-            .expect("The sender should only be None after de-initialization.")
-    }
-
-    const fn new(
-        sender: mpsc::UnboundedSender<AsyncTaskMessage>,
-        receiver: mpsc::UnboundedReceiver<()>,
-    ) -> Self {
-        Self {
-            sender: Some(sender),
-            receiver,
-        }
-    }
-}
-
-// #[gen_stub_pymethods]
-#[pymethods]
-impl PyAsyncTaskHandle {
-    #[new]
-    fn py_new(py: Python<'_>) -> PyResult<Self> {
-        use pyo3_async_runtimes::tokio::get_runtime;
-
-        // create communication channel TOWARDS our task
-        let (h_sender, t_receiver) = mpsc::unbounded_channel::<AsyncTaskMessage>();
-
-        // create communication channel FROM our task
-        let (t_sender, h_receiver) = mpsc::unbounded_channel::<()>();
-
-        // perform necessary setup within tokio context - or it crashes
-        let () = get_runtime().block_on(async { needs_tokio_runtime() });
-
-        // spawn tokio task with this thread's task-locals - without this, async callbacks on the new threads will not work!!
-        _ = get_runtime().spawn_with_scope(py, async move {
-            async_task(t_sender, t_receiver).await;
-        });
-        Ok(Self::new(h_sender, h_receiver))
-    }
-
-    /// NOTE: exceptions in callbacks are silently ignored until end of execution
-    fn add_sync_callback(
-        &self,
-        // #[gen_stub(override_type(
-        //     type_repr="collections.abc.Callable[[], None]",
-        //     imports=("collections.abc")
-        // ))]
-        callback: Py<PyAny>,
-    ) -> PyResult<()> {
-        // blocking call to async method -> can do non-blocking if needed
-        self.sender()
-            .send(AsyncTaskMessage::SyncCallback(Box::new(move || {
-                _ = Python::with_gil(|py| callback.call0(py).write_unraisable_with(py));
-            })))
-            .pyerr()?;
-        Ok(())
-    }
-
-    /// NOTE: exceptions in callbacks are silently ignored until end of execution
-    fn add_async_callback(
-        &self,
-        // #[gen_stub(override_type(
-        //     type_repr="collections.abc.Callable[[], collections.abc.Awaitable[None]]",
-        //     imports=("collections.abc")
-        // ))]
-        callback: Py<PyAny>,
-    ) -> PyResult<()> {
-        // blocking call to async method -> can do non-blocking if needed
-        self.sender()
-            .send(AsyncTaskMessage::AsyncCallback(Box::new(move || {
-                let c = Python::with_gil(|py| callback.clone_ref(py));
-                async move {
-                    if let Some(f) = Python::with_gil(|py| {
-                        let coroutine = c.call0(py).write_unraisable_with(py)?;
-                        pyo3_async_runtimes::tokio::into_future(coroutine.into_bound(py))
-                            .write_unraisable_with(py)
-                    }) {
-                        _ = f.await.write_unraisable();
-                    }
-                }
-                .boxed()
-            })))
-            .pyerr()?;
-        Ok(())
-    }
-
-    async fn receive_unit(&mut self) -> PyResult<()> {
-        self.receiver
-            .recv()
-            .await
-            .ok_or(PyErr::new::<PyRuntimeError, _>(
-                "cannot receive unit on closed channel",
-            ))
-    }
-
-    fn drain_units(&mut self) -> PyResult<i32> {
-        let mut cnt = 0;
-        loop {
-            match self.receiver.try_recv() {
-                Err(TryRecvError::Disconnected) => {
-                    return Err(PyErr::new::<PyRuntimeError, _>(
-                        "cannot receive unit on closed channel",
-                    ));
-                }
-                Err(TryRecvError::Empty) => return Ok(cnt),
-                Ok(()) => {
-                    cnt += 1;
-                    continue;
-                }
-            }
-        }
-    }
-
-    // #[gen_stub(skip)]
-    const fn __traverse__(&self, _visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
-        Ok(()) // This is needed purely so `__clear__` can work
-    }
-
-    // #[gen_stub(skip)]
-    fn __clear__(&mut self) {
-        // TODO: may or may not need to await a "kill-signal" oneshot channel message,
-        //       to ensure that the networking task is done BEFORE exiting the clear function...
-        //       but this may require GIL?? and it may not be safe to call GIL here??
-        self.sender = None; // Using Option<T> as a trick to force `sender` channel to be dropped
-    }
-}
-
-pub fn examples_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> {
-    m.add_class::<PyAsyncTaskHandle>()?;
-
-    Ok(())
-}
@@ -1,216 +1,42 @@
 //! TODO: crate documentation
-//!
-//! this is here as a placeholder documentation
-//!
-//!
+pub(crate) mod allow_threading;

-// enable Rust-unstable features for convenience
-#![feature(trait_alias)]
-#![feature(tuple_trait)]
-#![feature(unboxed_closures)]
-// #![feature(stmt_expr_attributes)]
-// #![feature(assert_matches)]
-// #![feature(async_fn_in_dyn_trait)]
-// #![feature(async_for_loop)]
-// #![feature(auto_traits)]
-// #![feature(negative_impls)]
-
-extern crate core;
-mod allow_threading;
-mod examples;
 pub(crate) mod networking;
-pub(crate) mod pylibp2p;
+pub(crate) mod take_once {
+    use std::sync::Mutex;
+
+    pub struct TakeOnce<T>(Mutex<Option<T>>);
+    impl<T> TakeOnce<T> {
+        pub fn new(t: T) -> Self {
+            Self(Mutex::new(Some(t)))
+        }
+        pub fn take(&self) -> Option<T> {
+            match self.0.try_lock() {
+                Ok(mut o) => o.take(),
+                Err(_) => None,
+            }
+        }
+    }
+}

-use crate::networking::networking_submodule;
-use crate::pylibp2p::ident::ident_submodule;
-use crate::pylibp2p::multiaddr::multiaddr_submodule;
-use pyo3::prelude::PyModule;
 use pyo3::prelude::*;
-use pyo3::{Bound, PyResult, pyclass, pymodule};
+
 use pyo3_stub_gen::define_stub_info_gatherer;

-/// Namespace for all the constants used by this crate.
-pub(crate) mod r#const {
-    pub const MPSC_CHANNEL_SIZE: usize = 1024;
-}
-
-/// Namespace for all the type/trait aliases used by this crate.
-pub(crate) mod alias {
-    use std::error::Error;
-    use std::marker::Tuple;
-
-    pub trait SendFn<Args: Tuple + Send + 'static, Output> =
-        Fn<Args, Output = Output> + Send + 'static;
-
-    pub type AnyError = Box<dyn Error + Send + Sync + 'static>;
-    pub type AnyResult<T> = Result<T, AnyError>;
-}
-
-/// Namespace for crate-wide extension traits/methods
-pub(crate) mod ext {
-    use crate::allow_threading::AllowThreads;
-    use extend::ext;
-    use pyo3::exceptions::{PyConnectionError, PyRuntimeError};
-    use pyo3::marker::Ungil;
-    use pyo3::types::PyBytes;
-    use pyo3::{Py, PyErr, PyResult, Python};
-    use tokio::runtime::Runtime;
-    use tokio::sync::mpsc;
-    use tokio::sync::mpsc::error::TryRecvError;
-    use tokio::task::JoinHandle;
-
-    #[ext(pub, name = ByteArrayExt)]
-    impl [u8] {
-        fn pybytes(&self) -> Py<PyBytes> {
-            Python::with_gil(|py| PyBytes::new(py, self).unbind())
-        }
-    }
-
-    #[ext(pub, name = ResultExt)]
-    impl<T, E> Result<T, E>
-    where
-        E: ToString,
-    {
-        fn pyerr(self) -> PyResult<T> {
-            self.map_err(|e| PyRuntimeError::new_err(e.to_string()))
-        }
-    }
-
-    pub trait FutureExt: Future + Sized {
-        /// SEE: https://pyo3.rs/v0.26.0/async-await.html#detaching-from-the-interpreter-across-await
-        fn allow_threads_py(self) -> AllowThreads<Self>
-        where
-            AllowThreads<Self>: Future,
-        {
-            AllowThreads::new(self)
-        }
-    }
-
-    impl<T: Future> FutureExt for T {}
-
-    #[ext(pub, name = PyErrExt)]
-    impl PyErr {
-        fn receiver_channel_closed() -> Self {
-            PyConnectionError::new_err("Receiver channel closed unexpectedly")
-        }
-    }
-
-    #[ext(pub, name = PyResultExt)]
-    impl<T> PyResult<T> {
-        fn write_unraisable(self) -> Option<T> {
-            Python::with_gil(|py| self.write_unraisable_with(py))
-        }
-
-        fn write_unraisable_with(self, py: Python<'_>) -> Option<T> {
-            match self {
-                Ok(v) => Some(v),
-                Err(e) => {
-                    // write error back to python
-                    e.write_unraisable(py, None);
-                    None
-                }
-            }
-        }
-    }
-
-    #[ext(pub, name = TokioRuntimeExt)]
-    impl Runtime {
-        fn spawn_with_scope<F>(&self, py: Python<'_>, future: F) -> PyResult<JoinHandle<F::Output>>
-        where
-            F: Future + Send + 'static,
-            F::Output: Send + 'static,
-        {
-            let locals = pyo3_async_runtimes::tokio::get_current_locals(py)?;
-            Ok(self.spawn(pyo3_async_runtimes::tokio::scope(locals, future)))
-        }
-    }
-
-    #[ext(pub, name = TokioMpscSenderExt)]
-    impl<T> mpsc::Sender<T> {
-        /// Sends a value, waiting until there is capacity.
-        ///
-        /// A successful send occurs when it is determined that the other end of the
-        /// channel has not hung up already. An unsuccessful send would be one where
-        /// the corresponding receiver has already been closed.
-        async fn send_py(&self, value: T) -> PyResult<()> {
-            self.send(value)
-                .await
-                .map_err(|_| PyErr::receiver_channel_closed())
-        }
-    }
-
-    #[ext(pub, name = TokioMpscReceiverExt)]
-    impl<T> mpsc::Receiver<T> {
-        /// Receives the next value for this receiver.
-        async fn recv_py(&mut self) -> PyResult<T> {
-            self.recv().await.ok_or_else(PyErr::receiver_channel_closed)
-        }
-
-        /// Receives at most `limit` values for this receiver and returns them.
-        ///
-        /// For `limit = 0`, an empty collection of messages will be returned immediately.
-        /// For `limit > 0`, if there are no messages in the channel's queue this method
-        /// will sleep until a message is sent.
-        async fn recv_many_py(&mut self, limit: usize) -> PyResult<Vec<T>> {
-            // get updates from receiver channel
-            let mut updates = Vec::with_capacity(limit);
-            let received = self.recv_many(&mut updates, limit).await;
-
-            // if we received zero items, then the channel was unexpectedly closed
-            if limit != 0 && received == 0 {
-                return Err(PyErr::receiver_channel_closed());
-            }
-
-            Ok(updates)
-        }
-
-        /// Tries to receive the next value for this receiver.
-        fn try_recv_py(&mut self) -> PyResult<Option<T>> {
-            match self.try_recv() {
-                Ok(v) => Ok(Some(v)),
-                Err(TryRecvError::Empty) => Ok(None),
-                Err(TryRecvError::Disconnected) => Err(PyErr::receiver_channel_closed()),
-            }
-        }
-    }
-}
-
-pub(crate) mod private {
-    use std::marker::Sized;
-
-    /// Sealed traits support
-    pub trait Sealed {}
-    impl<T: ?Sized> Sealed for T {}
-}
-
-/// A wrapper around [`Py`] that implements [`Clone`] using [`Python::with_gil`].
-#[repr(transparent)]
-pub(crate) struct ClonePy<T>(pub Py<T>);
-
-impl<T> Clone for ClonePy<T> {
-    fn clone(&self) -> Self {
-        Python::with_gil(|py| Self(self.0.clone_ref(py)))
-    }
-}
-
 /// A Python module implemented in Rust. The name of this function must match
 /// the `lib.name` setting in the `Cargo.toml`, else Python will not be able to
 /// import the module.
 #[pymodule(name = "exo_pyo3_bindings")]
-fn main_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
+pub fn networking_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
    // install logger
    pyo3_log::init();
+    // setup runtime
+    let mut builder = tokio::runtime::Builder::new_multi_thread();
+    builder.enable_all();
+    pyo3_async_runtimes::tokio::init(builder);

-    // TODO: for now this is all NOT a submodule, but figure out how to make the submodule system
-    //       work with maturin, where the types generate correctly, in the right folder, without
-    //       too many importing issues...
-    ident_submodule(m)?;
-    multiaddr_submodule(m)?;
-    networking_submodule(m)?;
-
-    // top-level constructs
-    // TODO: ...
-
+    m.add_class::<networking::PyPeer>()?;
+    m.add_class::<networking::PyKeypair>()?;
    Ok(())
 }

@@ -1,572 +1,214 @@
-#![allow(
-    clippy::multiple_inherent_impl,
-    clippy::unnecessary_wraps,
-    clippy::unused_self,
-    clippy::needless_pass_by_value
-)]
+use crate::allow_threading::AllowThreads;
+use crate::take_once::TakeOnce;

-use crate::r#const::MPSC_CHANNEL_SIZE;
-use crate::ext::{ByteArrayExt as _, FutureExt, PyErrExt as _};
-use crate::ext::{ResultExt as _, TokioMpscReceiverExt as _, TokioMpscSenderExt as _};
-use crate::pyclass;
-use crate::pylibp2p::ident::{PyKeypair, PyPeerId};
-use libp2p::futures::StreamExt as _;
-use libp2p::gossipsub::{IdentTopic, Message, MessageId, PublishError};
-use libp2p::swarm::SwarmEvent;
-use libp2p::{gossipsub, mdns};
-use networking::discovery;
-use networking::swarm::create_swarm;
-use pyo3::prelude::{PyModule, PyModuleMethods as _};
-use pyo3::types::PyBytes;
-use pyo3::{Bound, Py, PyErr, PyResult, PyTraverseError, PyVisit, Python, pymethods};
-use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pyclass_enum, gen_stub_pymethods};
-use std::net::IpAddr;
-use tokio::sync::{Mutex, mpsc, oneshot};
+use std::pin::pin;

-mod exception {
-    use pyo3::types::PyTuple;
-    use pyo3::{PyErrArguments, exceptions::PyException, prelude::*};
-    use pyo3_stub_gen::derive::*;
-
-    #[gen_stub_pyclass]
-    #[pyclass(frozen, extends=PyException, name="NoPeersSubscribedToTopicError")]
-    pub struct PyNoPeersSubscribedToTopicError {}
-
-    impl PyNoPeersSubscribedToTopicError {
-        const MSG: &'static str = "\
-        No peers are currently subscribed to receive messages on this topic. \
-        Wait for peers to subscribe or check your network connectivity.";
-
-        ///   Creates a new  [ `PyErr` ]  of this type.
-        ///
-        ///   [`PyErr`] :  https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html   "PyErr in pyo3"
-        pub(crate) fn new_err() -> PyErr {
-            PyErr::new::<Self, _>(()) // TODO: check if this needs to be replaced???
-        }
-    }
-
-    #[gen_stub_pymethods]
-    #[pymethods]
-    impl PyNoPeersSubscribedToTopicError {
-        #[new]
-        #[pyo3(signature = (*args))]
-        #[allow(unused_variables)]
-        pub(crate) fn new(args: &Bound<'_, PyTuple>) -> Self {
-            Self {}
-        }
-
-        fn __repr__(&self) -> String {
-            format!("PeerId(\"{}\")", Self::MSG)
-        }
-
-        fn __str__(&self) -> String {
-            Self::MSG.to_string()
-        }
-    }
-
-    #[gen_stub_pyclass]
-    #[pyclass(frozen, extends=PyException, name="AllQueuesFullError")]
-    pub struct PyAllQueuesFullError {}
-
-    impl PyAllQueuesFullError {
-        const MSG: &'static str =
-            "All libp2p peers are unresponsive, resend the message or reconnect.";
-
-        ///   Creates a new  [ `PyErr` ]  of this type.
-        ///
-        ///   [`PyErr`] :  https://docs.rs/pyo3/latest/pyo3/struct.PyErr.html   "PyErr in pyo3"
-        pub(crate) fn new_err() -> PyErr {
-            PyErr::new::<Self, _>(()) // TODO: check if this needs to be replaced???
-        }
-    }
-
-    #[gen_stub_pymethods]
-    #[pymethods]
-    impl PyAllQueuesFullError {
-        #[new]
-        #[pyo3(signature = (*args))]
-        #[allow(unused_variables)]
-        pub(crate) fn new(args: &Bound<'_, PyTuple>) -> Self {
-            Self {}
-        }
-
-        fn __repr__(&self) -> String {
-            format!("PeerId(\"{}\")", Self::MSG)
-        }
-
-        fn __str__(&self) -> String {
-            Self::MSG.to_string()
-        }
-    }
-}
-
-/// Connection or disconnection event discriminant type.
-#[gen_stub_pyclass_enum]
-#[pyclass(eq, eq_int, name = "ConnectionUpdateType")]
-#[derive(Debug, Clone, PartialEq)]
-enum PyConnectionUpdateType {
-    Connected = 0,
-    Disconnected,
-}
+use futures_lite::FutureExt;
+use libp2p::{gossipsub::PublishError, identity::Keypair};
+use networking::{FromSwarm, Peer, ToSwarm};
+use pyo3::{
+    coroutine::CancelHandle,
+    exceptions::{PyConnectionError, PyRuntimeError, PyValueError},
+    prelude::*,
+    types::PyBytes,
+};
+use pyo3_stub_gen::{
+    derive::{gen_methods_from_python, gen_stub_pyclass, gen_stub_pymethods},
+    inventory::submit,
+};
+use tokio::sync::{Mutex, mpsc};

 #[gen_stub_pyclass]
-#[pyclass(frozen, name = "ConnectionUpdate")]
-#[derive(Debug, Clone)]
-struct PyConnectionUpdate {
-    /// Whether this is a connection or disconnection event
-    #[pyo3(get)]
-    update_type: PyConnectionUpdateType,
+#[pyclass(name = "Keypair", frozen)]
+#[derive(Clone)]
+pub struct PyKeypair(Keypair);

-    /// Identity of the peer that we have connected to or disconnected from.
-    #[pyo3(get)]
-    peer_id: PyPeerId,
+#[gen_stub_pymethods]
+#[pymethods]
+impl PyKeypair {
+    /// Generate a new ed25519 keypair
+    #[staticmethod]
+    fn generate() -> Self {
+        Self(Keypair::generate_ed25519())
+    }

-    /// Remote connection's IPv4 address.
-    #[pyo3(get)]
-    remote_ipv4: String,
+    /// Decode a private key from a protobuf structure and parse it as a `Keypair`.
+    #[staticmethod]
+    fn from_protobuf_encoding(bytes: &Bound<'_, PyBytes>) -> Self {
+        let bytes = Vec::from(bytes.as_bytes());
+        Self(Keypair::from_protobuf_encoding(&bytes).expect("todo"))
+    }

-    /// Remote connection's TCP port.
-    #[pyo3(get)]
-    remote_tcp_port: u16,
-}
-
-enum ToTask {
-    GossipsubSubscribe {
-        topic: String,
-        result_tx: oneshot::Sender<PyResult<bool>>,
-    },
-    GossipsubUnsubscribe {
-        topic: String,
-        result_tx: oneshot::Sender<bool>,
-    },
-    GossipsubPublish {
-        topic: String,
-        data: Vec<u8>,
-        result_tx: oneshot::Sender<PyResult<MessageId>>,
-    },
-}
-
-#[allow(clippy::enum_glob_use)]
-async fn networking_task(
-    mut swarm: networking::swarm::Swarm,
-    mut to_task_rx: mpsc::Receiver<ToTask>,
-    connection_update_tx: mpsc::Sender<PyConnectionUpdate>,
-    gossipsub_message_tx: mpsc::Sender<(String, Vec<u8>)>,
-) {
-    use SwarmEvent::*;
-    use ToTask::*;
-    use mdns::Event::*;
-    use networking::swarm::BehaviourEvent::*;
-
-    log::info!("RUST: networking task started");
-
-    loop {
-        tokio::select! {
-            message = to_task_rx.recv() => {
-                // handle closed channel
-                let Some(message) = message else {
-                    log::info!("RUST: channel closed");
-                    break;
-                };
-
-                // dispatch incoming messages
-                match message {
-                    GossipsubSubscribe { topic, result_tx } => {
-                        // try to subscribe
-                        let result = swarm.behaviour_mut()
-                            .gossipsub.subscribe(&IdentTopic::new(topic));
-
-                        // send response oneshot
-                        if let Err(e) = result_tx.send(result.pyerr()) {
-                            log::error!("RUST: could not subscribe to gossipsub topic since channel already closed: {e:?}");
-                            continue;
-                        }
-                    }
-                    GossipsubUnsubscribe { topic, result_tx } => {
-                        // try to unsubscribe from the topic
-                        let result = swarm.behaviour_mut()
-                            .gossipsub.unsubscribe(&IdentTopic::new(topic));
-
-                        // send response oneshot (or exit if connection closed)
-                        if let Err(e) = result_tx.send(result) {
-                            log::error!("RUST: could not unsubscribe from gossipsub topic since channel already closed: {e:?}");
-                            continue;
-                        }
-                    }
-                    GossipsubPublish { topic, data, result_tx } => {
-                        // try to publish the data -> catch NoPeersSubscribedToTopic error & convert to correct exception
-                        let result = swarm.behaviour_mut().gossipsub.publish(
-                            IdentTopic::new(topic), data);
-                        let pyresult: PyResult<MessageId> = if let Err(PublishError::NoPeersSubscribedToTopic) = result {
-                            Err(exception::PyNoPeersSubscribedToTopicError::new_err())
-                        } else if let Err(PublishError::AllQueuesFull(_)) = result {
-                            Err(exception::PyAllQueuesFullError::new_err())
-                        } else {
-                            result.pyerr()
-                        };
-
-                        // send response oneshot (or exit if connection closed)
-                        if let Err(e) = result_tx.send(pyresult) {
-                            log::error!("RUST: could not publish gossipsub message since channel already closed: {e:?}");
-                            continue;
-                        }
-                    }
-                }
-            }
-
-            // architectural solution to this problem:
-            // create keep_alive behavior who's job it is to dial peers discovered by mDNS (and drop when expired)
-            //   -> it will emmit TRUE connected/disconnected events consumable elsewhere
-            //
-            // gossipsub will feed off-of dial attempts created by networking, and that will bootstrap its' peers list
-            // then for actual communication it will dial those peers if need-be
-            swarm_event = swarm.select_next_some() => {
-                match swarm_event {
-                    Behaviour(Gossipsub(gossipsub::Event::Message {
-                        message: Message {
-                            topic,
-                            data,
-                            ..
-                        },
-                        ..
-                    })) => {
-                        // topic-ID is just the topic hash!!! (since we used identity hasher)
-                        let message = (topic.into_string(), data);
-
-                        // send incoming message to channel (or exit if connection closed)
-                        if let Err(e) = gossipsub_message_tx.send(message).await {
-                            log::error!("RUST: could not send incoming gossipsub message since channel already closed: {e}");
-                            continue;
-                        }
-                    },
-                    Behaviour(Discovery(discovery::Event::ConnectionEstablished { peer_id, remote_ip, remote_tcp_port, .. })) => {
-                        // grab IPv4 string
-                        let remote_ipv4 = match remote_ip {
-                            IpAddr::V4(ip) => ip.to_string(),
-                            IpAddr::V6(ip) => {
-                                log::warn!("RUST: ignoring connection to IPv6 address: {ip}");
-                                continue;
-                            }
-                        };
-
-                        // send connection event to channel (or exit if connection closed)
-                        if let Err(e) = connection_update_tx.send(PyConnectionUpdate {
-                            update_type: PyConnectionUpdateType::Connected,
-                            peer_id: PyPeerId(peer_id),
-                            remote_ipv4,
-                            remote_tcp_port,
-                        }).await {
-                            log::error!("RUST: could not send connection update since channel already closed: {e}");
-                            continue;
-                        }
-                    },
-                    Behaviour(Discovery(discovery::Event::ConnectionClosed { peer_id, remote_ip, remote_tcp_port, .. })) => {
-                        // grab IPv4 string
-                        let remote_ipv4 = match remote_ip {
-                            IpAddr::V4(ip) => ip.to_string(),
-                            IpAddr::V6(ip) => {
-                                log::warn!("RUST: ignoring disconnection from IPv6 address: {ip}");
-                                continue;
-                            }
-                        };
-
-                        // send disconnection event to channel (or exit if connection closed)
-                        if let Err(e) = connection_update_tx.send(PyConnectionUpdate {
-                            update_type: PyConnectionUpdateType::Disconnected,
-                            peer_id: PyPeerId(peer_id),
-                            remote_ipv4,
-                            remote_tcp_port,
-                        }).await {
-                            log::error!("RUST: could not send connection update since channel already closed: {e}");
-                            continue;
-                        }
-                    },
-                    e => {
-                        log::info!("RUST: other event {e:?}");
-                    }
-                }
-            }
+    /// Encode a private key to a protobuf structure.
+    fn to_protobuf_encoding<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyBytes>> {
+        match self.0.to_protobuf_encoding() {
+            Ok(bytes) => Ok(PyBytes::new(py, &bytes)),
+            Err(e) => Err(PyValueError::new_err(e.to_string())),
        }
    }

-    log::info!("RUST: networking task stopped");
+    fn to_string(&self) -> String {
+        self.0.public().to_peer_id().to_base58()
+    }
 }

+struct PeerBuilder(
+    String,
+    Keypair,
+    mpsc::Sender<FromSwarm>,
+    mpsc::Receiver<ToSwarm>,
+);
+
 #[gen_stub_pyclass]
-#[pyclass(name = "NetworkingHandle")]
-#[derive(Debug)]
-struct PyNetworkingHandle {
-    // channels
-    to_task_tx: Option<mpsc::Sender<ToTask>>,
-    connection_update_rx: Mutex<mpsc::Receiver<PyConnectionUpdate>>,
-    gossipsub_message_rx: Mutex<mpsc::Receiver<(String, Vec<u8>)>>,
-}
-
-impl Drop for PyNetworkingHandle {
-    fn drop(&mut self) {
-        // TODO: may or may not need to await a "kill-signal" oneshot channel message,
-        //       to ensure that the networking task is done BEFORE exiting the clear function...
-        //       but this may require GIL?? and it may not be safe to call GIL here??
-        self.to_task_tx = None; // Using Option<T> as a trick to force channel to be dropped
-    }
-}
-
-#[allow(clippy::expect_used)]
-impl PyNetworkingHandle {
-    fn new(
-        to_task_tx: mpsc::Sender<ToTask>,
-        connection_update_rx: mpsc::Receiver<PyConnectionUpdate>,
-        gossipsub_message_rx: mpsc::Receiver<(String, Vec<u8>)>,
-    ) -> Self {
-        Self {
-            to_task_tx: Some(to_task_tx),
-            connection_update_rx: Mutex::new(connection_update_rx),
-            gossipsub_message_rx: Mutex::new(gossipsub_message_rx),
-        }
-    }
-
-    const fn to_task_tx(&self) -> &mpsc::Sender<ToTask> {
-        self.to_task_tx
-            .as_ref()
-            .expect("The sender should only be None after de-initialization.")
-    }
+#[pyclass]
+pub struct PyPeer {
+    peer: TakeOnce<PeerBuilder>,
+    to_swarm: mpsc::Sender<ToSwarm>,
+    from_swarm: Mutex<mpsc::Receiver<FromSwarm>>,
 }

 #[gen_stub_pymethods]
 #[pymethods]
-impl PyNetworkingHandle {
-    // NOTE: `async fn`s here that use `.await` will wrap the future in `.allow_threads_py()`
-    //       immediately beforehand to release the interpreter.
-    //       SEE: https://pyo3.rs/v0.26.0/async-await.html#detaching-from-the-interpreter-across-await
-
-    // ---- Lifecycle management methods ----
-
-    #[new]
-    fn py_new(identity: Bound<'_, PyKeypair>) -> PyResult<Self> {
-        use pyo3_async_runtimes::tokio::get_runtime;
-
-        // create communication channels
-        let (to_task_tx, to_task_rx) = mpsc::channel(MPSC_CHANNEL_SIZE);
-        let (connection_update_tx, connection_update_rx) = mpsc::channel(MPSC_CHANNEL_SIZE);
-        let (gossipsub_message_tx, gossipsub_message_rx) = mpsc::channel(MPSC_CHANNEL_SIZE);
-
-        // get identity
-        let identity = identity.borrow().0.clone();
-
-        // create networking swarm (within tokio context!! or it crashes)
-        let swarm = get_runtime()
-            .block_on(async { create_swarm(identity) })
-            .pyerr()?;
-
-        // spawn tokio task running the networking logic
-        get_runtime().spawn(async move {
-            networking_task(
-                swarm,
-                to_task_rx,
-                connection_update_tx,
-                gossipsub_message_tx,
-            )
-            .await;
-        });
-        Ok(Self::new(
-            to_task_tx,
-            connection_update_rx,
-            gossipsub_message_rx,
-        ))
+impl PyPeer {
+    #[staticmethod]
+    fn new(kp: PyKeypair, namespace: String) -> PyResult<Self> {
+        let (to_client, from_swarm) = mpsc::channel(1024);
+        let (to_swarm, from_client) = mpsc::channel(1024);
+        Ok(Self {
+            peer: TakeOnce::new(PeerBuilder(namespace, kp.0, to_client, from_client)),
+            to_swarm,
+            from_swarm: Mutex::new(from_swarm),
+        })
    }

    #[gen_stub(skip)]
-    const fn __traverse__(&self, _visit: PyVisit<'_>) -> Result<(), PyTraverseError> {
-        Ok(()) // This is needed purely so `__clear__` can work
+    async fn run(&self, #[pyo3(cancel_handle)] mut cancel: CancelHandle) -> PyResult<()> {
+        let builder = self
+            .peer
+            .take()
+            .ok_or_else(|| PyRuntimeError::new_err("tried to run peer twice"))?;
+        let jh = pyo3_async_runtimes::tokio::get_runtime()
+            .spawn(async move {
+                let mut peer =
+                    Peer::new(builder.0, builder.1, builder.2, builder.3).map_err(|_| {
+                        PyConnectionError::new_err("peer failed to listen on default address")
+                    })?;
+                peer.run()
+                    .await
+                    .map_err(|()| PyConnectionError::new_err("peer communication closed"))
+            })
+            .or(async {
+                cancel.cancelled().await;
+                Ok(Ok(()))
+            });
+        match AllowThreads(pin!(jh)).await {
+            Err(e) if e.is_cancelled() => Ok(()),
+            Err(e) if e.is_panic() => Err(PyRuntimeError::new_err(format!("tokio panic {e}"))),
+            Err(_) => unreachable!(),
+            Ok(res) => res,
+        }
+    }
+
+    async fn subscribe(&self, topic: String) -> PyResult<()> {
+        self.to_swarm
+            .send(ToSwarm::Subscribe(topic))
+            .await
+            .map_err(|_| PyRuntimeError::new_err("swarm communication closed"))
+    }
+    async fn unsubscribe(&self, topic: String) -> PyResult<()> {
+        self.to_swarm
+            .send(ToSwarm::Unsubscribe(topic))
+            .await
+            .map_err(|_| PyRuntimeError::new_err("swarm communication closed"))
+    }
+    async fn send(&self, topic: String, payload: Py<PyBytes>) -> PyResult<()> {
+        // this function attaches to the python interpreter synchronously to avoid holding the GIL
+        let bytes = Python::attach(|py| Vec::from(payload.bind(py).as_bytes()));
+        self.to_swarm
+            .send(ToSwarm::Message(topic, bytes))
+            .await
+            .map_err(|_| PyRuntimeError::new_err("swarm communication closed"))
    }

    #[gen_stub(skip)]
-    fn __clear__(&mut self) {
-        // TODO: may or may not need to await a "kill-signal" oneshot channel message,
-        //       to ensure that the networking task is done BEFORE exiting the clear function...
-        //       but this may require GIL?? and it may not be safe to call GIL here??
-        self.to_task_tx = None; // Using Option<T> as a trick to force channel to be dropped
+    async fn recv(
+        &self,
+        #[pyo3(cancel_handle)] mut cancel: CancelHandle,
+    ) -> PyResult<PySwarmEvent> {
+        loop {
+            return match AllowThreads(pin!(
+                self.from_swarm
+                    .try_lock()
+                    .map_err(|_| PyRuntimeError::new_err("tried to recv twice"))?
+                    .recv()
+                    .or(async {
+                        cancel.cancelled().await;
+                        None
+                    })
+            ))
+            .await
+            {
+                Some(FromSwarm::PublishError(p)) => match p {
+                    PublishError::AllQueuesFull(_) => {
+                        Err(PyConnectionError::new_err("swarm overloaded"))
+                    }
+                    PublishError::MessageTooLarge => {
+                        Err(PyValueError::new_err("message too large"))
+                    }
+                    PublishError::NoPeersSubscribedToTopic => {
+                        continue;
+                    }
+                    // TODO(evan): logs here
+                    _ => continue,
+                },
+                None => Err(PyRuntimeError::new_err("swarm communication closed")),
+                Some(fs) => Ok(PySwarmEvent(fs)),
+            };
+        }
    }
-
-    // ---- Connection update receiver methods ----
-
-    /// Receives the next `ConnectionUpdate` from networking.
-    async fn connection_update_recv(&self) -> PyResult<PyConnectionUpdate> {
-        self.connection_update_rx
-            .lock()
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-            .recv_py()
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-    }
-
-    /// Receives at most `limit` `ConnectionUpdate`s from networking and returns them.
-    ///
-    /// For `limit = 0`, an empty collection of `ConnectionUpdate`s will be returned immediately.
-    /// For `limit > 0`, if there are no `ConnectionUpdate`s in the channel's queue this method
-    /// will sleep until a `ConnectionUpdate`s is sent.
-    async fn connection_update_recv_many(&self, limit: usize) -> PyResult<Vec<PyConnectionUpdate>> {
-        self.connection_update_rx
-            .lock()
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-            .recv_many_py(limit)
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-    }
-
-    // TODO: rn this blocks main thread if anything else is awaiting the channel (bc its a mutex)
-    //       so its too dangerous to expose just yet. figure out a better semantics for handling this,
-    //       so things don't randomly block
-    // /// Tries to receive the next `ConnectionUpdate` from networking.
-    // fn connection_update_try_recv(&self) -> PyResult<Option<PyConnectionUpdate>> {
-    //     self.connection_update_rx.blocking_lock().try_recv_py()
-    // }
-    //
-    // /// Checks if the `ConnectionUpdate` channel is empty.
-    // fn connection_update_is_empty(&self) -> bool {
-    //     self.connection_update_rx.blocking_lock().is_empty()
-    // }
-    //
-    // /// Returns the number of `ConnectionUpdate`s in the channel.
-    // fn connection_update_len(&self) -> usize {
-    //     self.connection_update_rx.blocking_lock().len()
-    // }
-
-    // ---- Gossipsub management methods ----
-
-    /// Subscribe to a `GossipSub` topic.
-    ///
-    /// Returns `True` if the subscription worked. Returns `False` if we were already subscribed.
-    async fn gossipsub_subscribe(&self, topic: String) -> PyResult<bool> {
-        let (tx, rx) = oneshot::channel();
-
-        // send off request to subscribe
-        self.to_task_tx()
-            .send_py(ToTask::GossipsubSubscribe {
-                topic,
-                result_tx: tx,
-            })
-            .allow_threads_py() // allow-threads-aware async call
-            .await?;
-
-        // wait for response & return any errors
-        rx.allow_threads_py() // allow-threads-aware async call
-            .await
-            .map_err(|_| PyErr::receiver_channel_closed())?
-    }
-
-    /// Unsubscribes from a `GossipSub` topic.
-    ///
-    /// Returns `True` if we were subscribed to this topic. Returns `False` if we were not subscribed.
-    async fn gossipsub_unsubscribe(&self, topic: String) -> PyResult<bool> {
-        let (tx, rx) = oneshot::channel();
-
-        // send off request to unsubscribe
-        self.to_task_tx()
-            .send_py(ToTask::GossipsubUnsubscribe {
-                topic,
-                result_tx: tx,
-            })
-            .allow_threads_py() // allow-threads-aware async call
-            .await?;
-
-        // wait for response & convert any errors
-        rx.allow_threads_py() // allow-threads-aware async call
-            .await
-            .map_err(|_| PyErr::receiver_channel_closed())
-    }
-
-    /// Publishes a message with multiple topics to the `GossipSub` network.
-    ///
-    /// If no peers are found that subscribe to this topic, throws `NoPeersSubscribedToTopicError` exception.
-    async fn gossipsub_publish(&self, topic: String, data: Py<PyBytes>) -> PyResult<()> {
-        let (tx, rx) = oneshot::channel();
-
-        // send off request to subscribe
-        let data = Python::with_gil(|py| Vec::from(data.as_bytes(py)));
-        self.to_task_tx()
-            .send_py(ToTask::GossipsubPublish {
-                topic,
-                data,
-                result_tx: tx,
-            })
-            .allow_threads_py() // allow-threads-aware async call
-            .await?;
-
-        // wait for response & return any errors => ignore messageID for now!!!
-        let _ = rx
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-            .map_err(|_| PyErr::receiver_channel_closed())??;
-        Ok(())
-    }
-
-    // ---- Gossipsub message receiver methods ----
-
-    /// Receives the next message from the `GossipSub` network.
-    async fn gossipsub_recv(&self) -> PyResult<(String, Py<PyBytes>)> {
-        self.gossipsub_message_rx
-            .lock()
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-            .recv_py()
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-            .map(|(t, d)| (t, d.pybytes()))
-    }
-
-    /// Receives at most `limit` messages from the `GossipSub` network and returns them.
-    ///
-    /// For `limit = 0`, an empty collection of messages will be returned immediately.
-    /// For `limit > 0`, if there are no messages in the channel's queue this method
-    /// will sleep until a message is sent.
-    async fn gossipsub_recv_many(&self, limit: usize) -> PyResult<Vec<(String, Py<PyBytes>)>> {
-        Ok(self
-            .gossipsub_message_rx
-            .lock()
-            .allow_threads_py() // allow-threads-aware async call
-            .await
-            .recv_many_py(limit)
-            .allow_threads_py() // allow-threads-aware async call
-            .await?
-            .into_iter()
-            .map(|(t, d)| (t, d.pybytes()))
-            .collect())
-    }
-
-    // TODO: rn this blocks main thread if anything else is awaiting the channel (bc its a mutex)
-    //       so its too dangerous to expose just yet. figure out a better semantics for handling this,
-    //       so things don't randomly block
-    // /// Tries to receive the next message from the `GossipSub` network.
-    // fn gossipsub_try_recv(&self) -> PyResult<Option<(String, Py<PyBytes>)>> {
-    //     Ok(self
-    //         .gossipsub_message_rx
-    //         .blocking_lock()
-    //         .try_recv_py()?
-    //         .map(|(t, d)| (t, d.pybytes())))
-    // }
-    //
-    // /// Checks if the `GossipSub` message channel is empty.
-    // fn gossipsub_is_empty(&self) -> bool {
-    //     self.gossipsub_message_rx.blocking_lock().is_empty()
-    // }
-    //
-    // /// Returns the number of `GossipSub` messages in the channel.
-    // fn gossipsub_len(&self) -> usize {
-    //     self.gossipsub_message_rx.blocking_lock().len()
-    // }
 }

-pub fn networking_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> {
-    m.add_class::<exception::PyNoPeersSubscribedToTopicError>()?;
-    m.add_class::<exception::PyAllQueuesFullError>()?;
-
-    m.add_class::<PyConnectionUpdateType>()?;
-    m.add_class::<PyConnectionUpdate>()?;
-    m.add_class::<PyConnectionUpdateType>()?;
-    m.add_class::<PyNetworkingHandle>()?;
-
-    Ok(())
+// Manually submit the run()/recv() stub because the cancelhandle is poorly understood
+submit! {
+    gen_methods_from_python! {
+        r#"
+        class PyPeer:
+            async def run(self): ...
+            async def recv(self) -> PySwarmEvent: ...
+        "#
+    }
+}
+
+#[gen_stub_pyclass]
+#[pyclass]
+pub struct PySwarmEvent(FromSwarm);
+
+#[gen_stub_pymethods]
+#[pymethods]
+impl PySwarmEvent {
+    // probably a better way to do this, but...
+    fn downcast_discovered(&self) -> Option<String> {
+        if let FromSwarm::Discovered(peer_id) = self.0 {
+            Some(peer_id.to_base58())
+        } else {
+            None
+        }
+    }
+    fn downcast_expired(&self) -> Option<String> {
+        if let FromSwarm::Expired(peer_id) = self.0 {
+            Some(peer_id.to_base58())
+        } else {
+            None
+        }
+    }
+    fn downcast_message<'py>(
+        &self,
+        py: Python<'py>,
+    ) -> Option<(String, String, Bound<'py, PyBytes>)> {
+        if let FromSwarm::Message(peer_id, topic, data) = &self.0 {
+            Some((peer_id.to_base58(), topic.clone(), PyBytes::new(py, data)))
+        } else {
+            None
+        }
+    }
 }
@@ -1,159 +0,0 @@
-use crate::ext::ResultExt as _;
-use libp2p::PeerId;
-use libp2p::identity::Keypair;
-use pyo3::prelude::{PyBytesMethods as _, PyModule, PyModuleMethods as _};
-use pyo3::types::PyBytes;
-use pyo3::{Bound, PyResult, Python, pyclass, pymethods};
-use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods};
-
-/// Identity keypair of a node.
-#[gen_stub_pyclass]
-#[pyclass(name = "Keypair", frozen)]
-#[repr(transparent)]
-pub struct PyKeypair(pub Keypair);
-
-#[gen_stub_pymethods]
-#[pymethods]
-#[allow(clippy::needless_pass_by_value)]
-impl PyKeypair {
-    /// Generate a new Ed25519 keypair.
-    #[staticmethod]
-    fn generate_ed25519() -> Self {
-        Self(Keypair::generate_ed25519())
-    }
-
-    /// Generate a new ECDSA keypair.
-    #[staticmethod]
-    fn generate_ecdsa() -> Self {
-        Self(Keypair::generate_ecdsa())
-    }
-
-    /// Generate a new Secp256k1 keypair.
-    #[staticmethod]
-    fn generate_secp256k1() -> Self {
-        Self(Keypair::generate_secp256k1())
-    }
-
-    /// Decode a private key from a protobuf structure and parse it as a `Keypair`.
-    #[staticmethod]
-    fn from_protobuf_encoding(bytes: Bound<'_, PyBytes>) -> PyResult<Self> {
-        let bytes = Vec::from(bytes.as_bytes());
-        Ok(Self(Keypair::from_protobuf_encoding(&bytes).pyerr()?))
-    }
-
-    /// Decode an keypair from a DER-encoded secret key in PKCS#8 `PrivateKeyInfo`
-    /// format (i.e. unencrypted) as defined in [RFC5208].
-    ///
-    /// [RFC5208]: https://tools.ietf.org/html/rfc5208#section-5
-    #[staticmethod]
-    fn rsa_from_pkcs8(bytes: Bound<'_, PyBytes>) -> PyResult<Self> {
-        let mut bytes = Vec::from(bytes.as_bytes());
-        Ok(Self(Keypair::rsa_from_pkcs8(&mut bytes).pyerr()?))
-    }
-
-    /// Decode a keypair from a DER-encoded Secp256k1 secret key in an `ECPrivateKey`
-    /// structure as defined in [RFC5915].
-    ///
-    /// [RFC5915]: https://tools.ietf.org/html/rfc5915
-    #[staticmethod]
-    fn secp256k1_from_der(bytes: Bound<'_, PyBytes>) -> PyResult<Self> {
-        let mut bytes = Vec::from(bytes.as_bytes());
-        Ok(Self(Keypair::secp256k1_from_der(&mut bytes).pyerr()?))
-    }
-
-    #[staticmethod]
-    fn ed25519_from_bytes(bytes: Bound<'_, PyBytes>) -> PyResult<Self> {
-        let mut bytes = Vec::from(bytes.as_bytes());
-        Ok(Self(Keypair::ed25519_from_bytes(&mut bytes).pyerr()?))
-    }
-
-    /// Encode a private key as protobuf structure.
-    fn to_protobuf_encoding<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyBytes>> {
-        let bytes = self.0.to_protobuf_encoding().pyerr()?;
-        Ok(PyBytes::new(py, &bytes))
-    }
-
-    /// Convert the `Keypair` into the corresponding `PeerId`.
-    fn to_peer_id(&self) -> PyPeerId {
-        PyPeerId(self.0.public().to_peer_id())
-    }
-
-    // /// Hidden constructor for pickling support. TODO: figure out how to do pickling...
-    // #[gen_stub(skip)]
-    // #[new]
-    // fn py_new(bytes: Bound<'_, PyBytes>) -> PyResult<Self> {
-    //     Self::from_protobuf_encoding(bytes)
-    // }
-    //
-    // #[gen_stub(skip)]
-    // fn __setstate__(&mut self, state: Bound<'_, PyBytes>) -> PyResult<()> {
-    //     *self = Self::from_protobuf_encoding(state)?;
-    //     Ok(())
-    // }
-    //
-    // #[gen_stub(skip)]
-    // fn __getstate__<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyBytes>> {
-    //     self.to_protobuf_encoding(py)
-    // }
-    //
-    // #[gen_stub(skip)]
-    // pub fn __getnewargs__<'py>(&self, py: Python<'py>) -> PyResult<(Bound<'py, PyBytes>,)> {
-    //     Ok((self.to_protobuf_encoding(py)?,))
-    // }
-}
-
-/// Identifier of a peer of the network.
-///
-/// The data is a `CIDv0` compatible multihash of the protobuf encoded public key of the peer
-/// as specified in [specs/peer-ids](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md).
-#[gen_stub_pyclass]
-#[pyclass(name = "PeerId", frozen)]
-#[derive(Debug, Clone)]
-#[repr(transparent)]
-pub struct PyPeerId(pub PeerId);
-
-#[gen_stub_pymethods]
-#[pymethods]
-#[allow(clippy::needless_pass_by_value)]
-impl PyPeerId {
-    /// Generates a random peer ID from a cryptographically secure PRNG.
-    ///
-    /// This is useful for randomly walking on a DHT, or for testing purposes.
-    #[staticmethod]
-    fn random() -> Self {
-        Self(PeerId::random())
-    }
-
-    /// Parses a `PeerId` from bytes.
-    #[staticmethod]
-    fn from_bytes(bytes: Bound<'_, PyBytes>) -> PyResult<Self> {
-        let bytes = Vec::from(bytes.as_bytes());
-        Ok(Self(PeerId::from_bytes(&bytes).pyerr()?))
-    }
-
-    /// Returns a raw bytes representation of this `PeerId`.
-    fn to_bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
-        let bytes = self.0.to_bytes();
-        PyBytes::new(py, &bytes)
-    }
-
-    /// Returns a base-58 encoded string of this `PeerId`.
-    fn to_base58(&self) -> String {
-        self.0.to_base58()
-    }
-
-    fn __repr__(&self) -> String {
-        format!("PeerId({})", self.to_base58())
-    }
-
-    fn __str__(&self) -> String {
-        self.to_base58()
-    }
-}
-
-pub fn ident_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> {
-    m.add_class::<PyKeypair>()?;
-    m.add_class::<PyPeerId>()?;
-
-    Ok(())
-}
@@ -1,8 +0,0 @@
-//! A module for exposing Rust's libp2p datatypes over Pyo3
-//!
-//! TODO: right now we are coupled to libp2p's identity, but eventually we want to create our own
-//!       independent identity type of some kind or another. This may require handshaking.
-//!
-
-pub mod ident;
-pub mod multiaddr;
@@ -1,81 +0,0 @@
-use crate::ext::ResultExt as _;
-use libp2p::Multiaddr;
-use pyo3::prelude::{PyBytesMethods as _, PyModule, PyModuleMethods as _};
-use pyo3::types::PyBytes;
-use pyo3::{Bound, PyResult, Python, pyclass, pymethods};
-use pyo3_stub_gen::derive::{gen_stub_pyclass, gen_stub_pymethods};
-use std::str::FromStr as _;
-
-/// Representation of a Multiaddr.
-#[gen_stub_pyclass]
-#[pyclass(name = "Multiaddr", frozen)]
-#[derive(Debug, Clone)]
-#[repr(transparent)]
-pub struct PyMultiaddr(pub Multiaddr);
-
-#[gen_stub_pymethods]
-#[pymethods]
-#[allow(clippy::needless_pass_by_value)]
-impl PyMultiaddr {
-    /// Create a new, empty multiaddress.
-    #[staticmethod]
-    fn empty() -> Self {
-        Self(Multiaddr::empty())
-    }
-
-    /// Create a new, empty multiaddress with the given capacity.
-    #[staticmethod]
-    fn with_capacity(n: usize) -> Self {
-        Self(Multiaddr::with_capacity(n))
-    }
-
-    /// Parse a `Multiaddr` value from its byte slice representation.
-    #[staticmethod]
-    fn from_bytes(bytes: Bound<'_, PyBytes>) -> PyResult<Self> {
-        let bytes = Vec::from(bytes.as_bytes());
-        Ok(Self(Multiaddr::try_from(bytes).pyerr()?))
-    }
-
-    /// Parse a `Multiaddr` value from its string representation.
-    #[staticmethod]
-    fn from_string(string: String) -> PyResult<Self> {
-        Ok(Self(Multiaddr::from_str(&string).pyerr()?))
-    }
-
-    /// Return the length in bytes of this multiaddress.
-    fn len(&self) -> usize {
-        self.0.len()
-    }
-
-    /// Returns true if the length of this multiaddress is 0.
-    fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-
-    /// Return a copy of this [`Multiaddr`]'s byte representation.
-    fn to_bytes<'py>(&self, py: Python<'py>) -> Bound<'py, PyBytes> {
-        let bytes = self.0.to_vec();
-        PyBytes::new(py, &bytes)
-    }
-
-    /// Convert a Multiaddr to a string.
-    fn to_string(&self) -> String {
-        self.0.to_string()
-    }
-
-    #[gen_stub(skip)]
-    fn __repr__(&self) -> String {
-        format!("Multiaddr({})", self.0)
-    }
-
-    #[gen_stub(skip)]
-    fn __str__(&self) -> String {
-        self.to_string()
-    }
-}
-
-pub fn multiaddr_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> {
-    m.add_class::<PyMultiaddr>()?;
-
-    Ok(())
-}
@@ -1,54 +0,0 @@
-#[cfg(test)]
-mod tests {
-    use core::mem::drop;
-    use core::option::Option::Some;
-    use core::time::Duration;
-    use tokio;
-    use tokio::sync::mpsc;
-
-    #[tokio::test]
-    async fn test_drop_channel() {
-        struct Ping;
-
-        let (tx, mut rx) = mpsc::channel::<Ping>(10);
-
-        let _ = tokio::spawn(async move {
-            println!("TASK: entered");
-
-            loop {
-                tokio::select! {
-                    result = rx.recv() => {
-                        match result {
-                            Some(_) => {
-                                println!("TASK: pinged");
-                            }
-                            None => {
-                                println!("TASK: closing channel");
-                                break;
-                            }
-                        }
-                    }
-                    _ = tokio::time::sleep(Duration::from_secs_f32(0.1)) => {
-                        println!("TASK: heartbeat");
-                    }
-                }
-            }
-
-            println!("TASK: exited");
-        });
-
-        let tx2 = tx.clone();
-
-        tokio::time::sleep(Duration::from_secs_f32(0.11)).await;
-
-        tx.send(Ping).await.expect("Should not fail");
-        drop(tx);
-
-        tokio::time::sleep(Duration::from_secs_f32(0.11)).await;
-
-        tx2.send(Ping).await.expect("Should not fail");
-        drop(tx2);
-
-        tokio::time::sleep(Duration::from_secs_f32(0.11)).await;
-    }
-}
@@ -13,32 +13,14 @@ path = "src/lib.rs"
 workspace = true

 [dependencies]
-# datastructures
-either = { workspace = true }
-
-# macro dependencies
-extend = { workspace = true }
-delegate = { workspace = true }
-impl-trait-for-tuples = { workspace = true }
-derive_more = { workspace = true }
-
 # async
 tokio = { workspace = true, features = ["full"] }
-futures = { workspace = true }
-futures-timer = { workspace = true }

 # utility dependencies
-util = { workspace = true }
-thiserror = { workspace = true }
-#internment = { workspace = true }
-#recursion = { workspace = true }
-#generativity = { workspace = true }
-#itertools = { workspace = true }
 tracing-subscriber = { version = "0.3.19", features = ["default", "env-filter"] }
-keccak-const = { workspace = true }

 # tracing/logging
 log = { workspace = true }

 # networking
-libp2p = { workspace = true, features = ["full"] }
+libp2p = { workspace = true, features = ["full"] }
@@ -1,6 +1,6 @@
-use futures::stream::StreamExt as _;
-use libp2p::{gossipsub, identity, swarm::SwarmEvent};
-use networking::{discovery, swarm};
+use libp2p::identity;
+use networking::{self, FromSwarm, ToSwarm};
+use tokio::sync::mpsc;
 use tokio::{io, io::AsyncBufReadExt as _, select};
 use tracing_subscriber::EnvFilter;
 use tracing_subscriber::filter::LevelFilter;
@@ -12,63 +12,51 @@ async fn main() {
        .try_init();

    // Configure swarm
-    let mut swarm =
-        swarm::create_swarm(identity::Keypair::generate_ed25519()).expect("Swarm creation failed");
+    let (to_client, mut from_swarm) = mpsc::channel(20);
+    let (to_swarm, from_client) = mpsc::channel(20);
+    let mut peer = networking::Peer::new(
+        "chatroom!".to_string(),
+        identity::Keypair::generate_ed25519(),
+        to_client,
+        from_client,
+    )
+    .expect("listen error");

    // Create a Gossipsub topic & subscribe
-    let topic = gossipsub::IdentTopic::new("test-net");
-    swarm
-        .behaviour_mut()
-        .gossipsub
-        .subscribe(&topic)
-        .expect("Subscribing to topic failed");
-
    // Read full lines from stdin
    let mut stdin = io::BufReader::new(io::stdin()).lines();
    println!("Enter messages via STDIN and they will be sent to connected peers using Gossipsub");

+    let jh = tokio::spawn(async move { peer.run().await });
+    _ = to_swarm
+        .send(ToSwarm::Subscribe("chatting".to_string()))
+        .await;
+
    // Kick it off
    loop {
        select! {
            // on gossipsub outgoing
            Ok(Some(line)) = stdin.next_line() => {
-                if let Err(e) = swarm
-                    .behaviour_mut().gossipsub
-                    .publish(topic.clone(), line.as_bytes()) {
-                    println!("Publish error: {e:?}");
-                }
+                _ = to_swarm.send(ToSwarm::Message("chatting".to_string(), line.into_bytes())).await;
            }
-            event = swarm.select_next_some() => match event {
+            event = from_swarm.recv() => match event {
                // on gossipsub incoming
-                SwarmEvent::Behaviour(swarm::BehaviourEvent::Gossipsub(gossipsub::Event::Message {
-                    propagation_source: peer_id,
-                    message_id: id,
-                    message,
-                })) => println!(
-                        "\n\nGot message: '{}' with id: {id} from peer: {peer_id}\n\n",
-                        String::from_utf8_lossy(&message.data),
+                Some(FromSwarm::Message(peer_id,_, data)) => println!(
+                        "\n\nGot message: '{}' from peer: {peer_id}\n\n",
+                        String::from_utf8_lossy(&data),
                    ),

                // on discovery
-                SwarmEvent::Behaviour(swarm::BehaviourEvent::Discovery(e)) => match e {
-                    discovery::Event::ConnectionEstablished {
-                        peer_id, connection_id, remote_ip, remote_tcp_port
-                    } => {
-                        println!("\n\nConnected to: {peer_id}; connection ID: {connection_id}; remote IP: {remote_ip}; remote TCP port: {remote_tcp_port}\n\n");
-                    }
-                    discovery::Event::ConnectionClosed {
-                        peer_id, connection_id, remote_ip, remote_tcp_port
-                    } => {
-                        eprintln!("\n\nDisconnected from: {peer_id}; connection ID: {connection_id}; remote IP: {remote_ip}; remote TCP port: {remote_tcp_port}\n\n");
-                    }
+                Some(FromSwarm::Discovered(peer_id)) => {
+                    println!("\n\nConnected to: {peer_id}\n\n");
                }
-
-                // ignore outgoing errors: those are normal
-                e@SwarmEvent::OutgoingConnectionError { .. } => { log::debug!("Outgoing connection error: {e:?}"); }
-
-                // otherwise log any other event
-                e => { log::info!("Other event {e:?}"); }
+                Some(FromSwarm::Expired(peer_id)) => {
+                    println!("\n\nDisconnected from: {peer_id}\n\n");
+                }
+                Some(FromSwarm::PublishError(e)) => eprintln!("\n\nError {e:?}\n\n"),
+                None => break,
            }
        }
    }
+    _ = jh.await;
 }
@@ -1,127 +0,0 @@
-// Copyright 2018 Parity Technologies (UK) Ltd.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a
-// copy of this software and associated documentation files (the "Software"),
-// to deal in the Software without restriction, including without limitation
-// the rights to use, copy, modify, merge, publish, distribute, sublicense,
-// and/or sell copies of the Software, and to permit persons to whom the
-// Software is furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in
-// all copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-// DEALINGS IN THE SOFTWARE.
-
-use futures::stream::StreamExt;
-use libp2p::{
-    gossipsub, mdns, noise,
-    swarm::{NetworkBehaviour, SwarmEvent},
-    tcp, yamux,
-};
-use std::time::Duration;
-use std::{error::Error, hash::Hash};
-use tokio::{io, io::AsyncBufReadExt, select};
-use tracing_subscriber::EnvFilter;
-
-// We create a custom network behaviour that combines Gossipsub and Mdns.
-#[derive(NetworkBehaviour)]
-struct MyBehaviour {
-    gossipsub: gossipsub::Behaviour,
-    mdns: mdns::tokio::Behaviour,
-}
-
-#[tokio::main]
-async fn main() -> Result<(), Box<dyn Error>> {
-    let _ = tracing_subscriber::fmt()
-        .with_env_filter(EnvFilter::from_default_env())
-        .try_init();
-
-    let mut swarm = libp2p::SwarmBuilder::with_new_identity()
-        .with_tokio()
-        .with_tcp(
-            tcp::Config::default(),
-            noise::Config::new,
-            yamux::Config::default,
-        )?
-        .with_behaviour(|key| {
-            // Set a custom gossipsub configuration
-            let gossipsub_config = gossipsub::ConfigBuilder::default()
-                .heartbeat_interval(Duration::from_secs(10))
-                .validation_mode(gossipsub::ValidationMode::Strict) // This sets the kind of message validation. The default is Strict (enforce message signing)
-                .build()
-                .map_err(io::Error::other)?; // Temporary hack because `build` does not return a proper `std::error::Error`.
-
-            // build a gossipsub network behaviour
-            let gossipsub = gossipsub::Behaviour::new(
-                gossipsub::MessageAuthenticity::Signed(key.clone()),
-                gossipsub_config,
-            )?;
-
-            let mdns =
-                mdns::tokio::Behaviour::new(mdns::Config::default(), key.public().to_peer_id())?;
-            Ok(MyBehaviour { gossipsub, mdns })
-        })?
-        .build();
-
-    println!("Running swarm with identity {}", swarm.local_peer_id());
-
-    // Create a Gossipsub topic
-    let topic = gossipsub::IdentTopic::new("test-net");
-    // subscribes to our topic
-    swarm.behaviour_mut().gossipsub.subscribe(&topic)?;
-
-    // Read full lines from stdin
-    let mut stdin = io::BufReader::new(io::stdin()).lines();
-
-    // Listen on all interfaces and whatever port the OS assigns
-    swarm.listen_on("/ip4/0.0.0.0/tcp/0".parse()?)?;
-
-    println!("Enter messages via STDIN and they will be sent to connected peers using Gossipsub");
-
-    // Kick it off
-    loop {
-        select! {
-            Ok(Some(line)) = stdin.next_line() => {
-                if let Err(e) = swarm
-                    .behaviour_mut().gossipsub
-                    .publish(topic.clone(), line.as_bytes()) {
-                    println!("Publish error: {e:?}");
-                }
-            }
-            event = swarm.select_next_some() => match event {
-                SwarmEvent::Behaviour(MyBehaviourEvent::Mdns(mdns::Event::Discovered(list))) => {
-                    for (peer_id, multiaddr) in list {
-                        println!("mDNS discovered a new peer: {peer_id} on {multiaddr}");
-                        swarm.behaviour_mut().gossipsub.add_explicit_peer(&peer_id);
-                    }
-                },
-                SwarmEvent::Behaviour(MyBehaviourEvent::Mdns(mdns::Event::Expired(list))) => {
-                    for (peer_id, multiaddr) in list {
-                        println!("mDNS discover peer has expired: {peer_id} on {multiaddr}");
-                        swarm.behaviour_mut().gossipsub.remove_explicit_peer(&peer_id);
-                    }
-                },
-                SwarmEvent::Behaviour(MyBehaviourEvent::Gossipsub(gossipsub::Event::Message {
-                    propagation_source: peer_id,
-                    message_id: id,
-                    message,
-                })) => println!(
-                        "Got message: '{}' with id: {id} from peer: {peer_id}",
-                        String::from_utf8_lossy(&message.data),
-                    ),
-                SwarmEvent::NewListenAddr { address, .. } => {
-                    println!("Local node is listening on {address}");
-                }
-                e => {
-                    println!("Other swarm event: {:?}", e);
-                }
-            }
-        }
-    }
-}
@@ -1,44 +0,0 @@
-https://github.com/ml-explore/mlx/commit/3fe98bacc7640d857acf3539f1d21b47a32e5609
-^raw sockets distributed -> `<net/ndrv.h>` -> https://newosxbook.com/code/xnu-3247.1.106/bsd/net/ndrv.h.auto.html
--> header file for a networking component found in the macOS kernel (XNU) that defines structures for network device driver registration, specifically the ndrv_demux_desc and ndrv_protocol_desc structures used for demultiplexing protocol data at the network interface level. It specifies how to describe protocol data, such as an Ethernet type or a SNAP header, and how to associate these descriptions with a specific protocol family to receive matching packets.
--> Used to bind an NDRV socket so that packets that match given protocol demux descriptions can be received.
--> An NDRV socket is a special kind of socket in the Darwin/macOS operating system's XNU kernel, used for low-level network packet manipulation and binding to specific protocols for packet processing. It allows user-space applications or drivers to directly write Layer 2 (L2) network packets or interact with the network stack at a lower level, often by binding to protocol descriptors like the ndrv_protocol_desc. This type of socket is used for functions such as capturing and injecting packets, especially in network infrastructure software like routers or for kernel-level network monitoring and security tools.
--> also called PF_NDRV sockets --> https://newosxbook.com/bonus/vol1ch16.html
----> they are conceptually similar to https://scapy.disruptivelabs.in/networking/socket-interface PF_RAW or PF_PACKET
-
-https://stackoverflow.com/questions/17169298/af-packet-on-osx
-^AF_PACKET duplicates the packets as soon as it receives them from the physical layer (for incoming packets) or just before sending them out to the physical layer (for outgoing packets). -> this is on Linux only
-^it doesn't exist on OS X so you can use /dev/bpfX (Berkeley Packet Filter) for sniffing
-
-https://www.unix.com/man_page/mojave/4/ip/
-^OS X manpages for IP
-
-https://developer.apple.com/documentation/kernel/implementing_drivers_system_extensions_and_kexts
-^driver kit, system extensions & kexts for macOS
-
----
-
-To set up a Linux system to use a Thunderbolt connection as a network device, connect the two computers with a Thunderbolt cable, load the thunderbolt-net kernel module (usually automatic but modprobe is an option for manual loading), and then the operating system will create virtual Ethernet interfaces (e.g., thunderbolt0) for networking. You can then use standard tools like ifconfig or your desktop environment's network manager to configure these new interfaces for a link-local network.
--> https://gist.github.com/geosp/80fbd39e617b7d1d9421683df4ea224a
----> here is a guide on how to set up thunderbolt-ethernet on linux
----> I may be able to steal the thunderbolt-net code ideas to implement a kernel module for MacOS
-
-https://chatgpt.com/s/t_68af8e41a8548191993281a014f846a7
-^GPT discussion about making socket interface
-
-https://chatgpt.com/s/t_68afb798a85c8191973c02a0fa7a48a3 --> link-local address,,??
-https://chatgpt.com/s/t_68afb02987e08191b2b0044d3667ece2
-^GPT discussion about accessing TB on MacOS low level interactions
-
--------------------------------
-
-https://www.intel.com/content/www/us/en/support/articles/000098893/software.html
-^Thunderbolt Share & Thunderbolt Networking Mode => intel's equivalent of thunderbolt bridge
-
-
---------------------------------
-
-https://www.zerotier.com/blog/how-zerotier-eliminated-kernel-extensions-on-macos/
-->fake ethernet devices on MacOS -> omg??? we can detect thunderbolt bridge, then bind to it, then re-expose it as fake ethernet??
-->ps: https://chatgpt.com/s/t_68afb2b25fb881919526763fb5d7359c, AF/PF_NDRV are one and the same!!!
-->https://github.com/zerotier/ZeroTierOne/blob/dev/osdep/MacEthernetTapAgent.c
@@ -1,383 +0,0 @@
-use crate::ext::MultiaddrExt;
-use crate::keep_alive;
-use delegate::delegate;
-use either::Either;
-use futures::FutureExt;
-use futures_timer::Delay;
-use libp2p::core::transport::PortUse;
-use libp2p::core::{ConnectedPoint, Endpoint};
-use libp2p::swarm::behaviour::ConnectionEstablished;
-use libp2p::swarm::dial_opts::DialOpts;
-use libp2p::swarm::{
-    CloseConnection, ConnectionClosed, ConnectionDenied, ConnectionHandler,
-    ConnectionHandlerSelect, ConnectionId, FromSwarm, NetworkBehaviour, THandler, THandlerInEvent,
-    THandlerOutEvent, ToSwarm, dummy,
-};
-use libp2p::{Multiaddr, PeerId, identity, mdns};
-use std::collections::{BTreeSet, HashMap};
-use std::convert::Infallible;
-use std::io;
-use std::net::IpAddr;
-use std::task::{Context, Poll};
-use std::time::Duration;
-use util::wakerdeque::WakerDeque;
-
-const RETRY_CONNECT_INTERVAL: Duration = Duration::from_secs(5);
-
-mod managed {
-    use libp2p::swarm::NetworkBehaviour;
-    use libp2p::{identity, mdns, ping};
-    use std::io;
-    use std::time::Duration;
-
-    const MDNS_RECORD_TTL: Duration = Duration::from_secs(2_500);
-    const MDNS_QUERY_INTERVAL: Duration = Duration::from_secs(1_500);
-    const PING_TIMEOUT: Duration = Duration::from_millis(2_500);
-    const PING_INTERVAL: Duration = Duration::from_millis(2_500);
-
-    #[derive(NetworkBehaviour)]
-    pub struct Behaviour {
-        mdns: mdns::tokio::Behaviour,
-        ping: ping::Behaviour,
-    }
-
-    impl Behaviour {
-        pub fn new(keypair: &identity::Keypair) -> io::Result<Self> {
-            Ok(Self {
-                mdns: mdns_behaviour(keypair)?,
-                ping: ping_behaviour(),
-            })
-        }
-    }
-
-    fn mdns_behaviour(keypair: &identity::Keypair) -> io::Result<mdns::tokio::Behaviour> {
-        use mdns::{Config, tokio};
-
-        // mDNS config => enable IPv6
-        let mdns_config = Config {
-            ttl: MDNS_RECORD_TTL,
-            query_interval: MDNS_QUERY_INTERVAL,
-
-            // enable_ipv6: true, // TODO: for some reason, TCP+mDNS don't work well with ipv6?? figure out how to make work
-            ..Default::default()
-        };
-
-        let mdns_behaviour = tokio::Behaviour::new(mdns_config, keypair.public().to_peer_id());
-        Ok(mdns_behaviour?)
-    }
-
-    fn ping_behaviour() -> ping::Behaviour {
-        ping::Behaviour::new(
-            ping::Config::new()
-                .with_timeout(PING_TIMEOUT)
-                .with_interval(PING_INTERVAL),
-        )
-    }
-}
-
-/// Events for when a listening connection is truly established and truly closed.
-#[derive(Debug, Clone)]
-pub enum Event {
-    ConnectionEstablished {
-        peer_id: PeerId,
-        connection_id: ConnectionId,
-        remote_ip: IpAddr,
-        remote_tcp_port: u16,
-    },
-    ConnectionClosed {
-        peer_id: PeerId,
-        connection_id: ConnectionId,
-        remote_ip: IpAddr,
-        remote_tcp_port: u16,
-    },
-}
-
-/// Discovery behavior that wraps mDNS to produce truly discovered durable peer-connections.
-///
-/// The behaviour operates as such:
-///  1) All true (listening) connections/disconnections are tracked, emitting corresponding events
-///     to the swarm.
-///  1) mDNS discovered/expired peers are tracked; discovered but not connected peers are dialed
-///     immediately, and expired but connected peers are disconnected from immediately.
-///  2) Every fixed interval: discovered but not connected peers are dialed, and expired but
-///     connected peers are disconnected from.
-pub struct Behaviour {
-    // state-tracking for managed behaviors & mDNS-discovered peers
-    managed: managed::Behaviour,
-    mdns_discovered: HashMap<PeerId, BTreeSet<Multiaddr>>,
-
-    retry_delay: Delay, // retry interval
-
-    // pending events to emmit => waker-backed Deque to control polling
-    pending_events: WakerDeque<ToSwarm<Event, Infallible>>,
-}
-
-impl Behaviour {
-    pub fn new(keypair: &identity::Keypair) -> io::Result<Self> {
-        Ok(Self {
-            managed: managed::Behaviour::new(keypair)?,
-            mdns_discovered: HashMap::new(),
-            retry_delay: Delay::new(RETRY_CONNECT_INTERVAL),
-            pending_events: WakerDeque::new(),
-        })
-    }
-
-    fn dial(&mut self, peer_id: PeerId, addr: Multiaddr) {
-        self.pending_events.push_back(ToSwarm::Dial {
-            opts: DialOpts::peer_id(peer_id).addresses(vec![addr]).build(),
-        })
-    }
-
-    fn close_connection(&mut self, peer_id: PeerId, connection: ConnectionId) {
-        // push front to make this IMMEDIATE
-        self.pending_events.push_front(ToSwarm::CloseConnection {
-            peer_id,
-            connection: CloseConnection::One(connection),
-        })
-    }
-
-    fn handle_mdns_discovered(&mut self, peers: Vec<(PeerId, Multiaddr)>) {
-        for (p, ma) in peers {
-            self.dial(p, ma.clone()); // always connect
-
-            // get peer's multi-addresses or insert if missing
-            let Some(mas) = self.mdns_discovered.get_mut(&p) else {
-                self.mdns_discovered.insert(p, BTreeSet::from([ma]));
-                continue;
-            };
-
-            // multiaddress should never already be present - else something has gone wrong
-            let is_new_addr = mas.insert(ma);
-            assert!(is_new_addr, "cannot discover a discovered peer");
-        }
-    }
-
-    fn handle_mdns_expired(&mut self, peers: Vec<(PeerId, Multiaddr)>) {
-        for (p, ma) in peers {
-            // at this point, we *must* have the peer
-            let mas = self
-                .mdns_discovered
-                .get_mut(&p)
-                .expect("nonexistent peer cannot expire");
-
-            // at this point, we *must* have the multiaddress
-            let was_present = mas.remove(&ma);
-            assert!(was_present, "nonexistent multiaddress cannot expire");
-
-            // if empty, remove the peer-id entirely
-            if mas.is_empty() {
-                self.mdns_discovered.remove(&p);
-            }
-        }
-    }
-
-    fn on_connection_established(
-        &mut self,
-        peer_id: PeerId,
-        connection_id: ConnectionId,
-        remote_ip: IpAddr,
-        remote_tcp_port: u16,
-    ) {
-        // send out connected event
-        self.pending_events
-            .push_back(ToSwarm::GenerateEvent(Event::ConnectionEstablished {
-                peer_id,
-                connection_id,
-                remote_ip,
-                remote_tcp_port,
-            }));
-    }
-
-    fn on_connection_closed(
-        &mut self,
-        peer_id: PeerId,
-        connection_id: ConnectionId,
-        remote_ip: IpAddr,
-        remote_tcp_port: u16,
-    ) {
-        // send out disconnected event
-        self.pending_events
-            .push_back(ToSwarm::GenerateEvent(Event::ConnectionClosed {
-                peer_id,
-                connection_id,
-                remote_ip,
-                remote_tcp_port,
-            }));
-    }
-}
-
-impl NetworkBehaviour for Behaviour {
-    type ConnectionHandler =
-        ConnectionHandlerSelect<dummy::ConnectionHandler, THandler<managed::Behaviour>>;
-    type ToSwarm = Event;
-
-    // simply delegate to underlying mDNS behaviour
-
-    delegate! {
-        to self.managed {
-            fn handle_pending_inbound_connection(&mut self, connection_id: ConnectionId, local_addr: &Multiaddr, remote_addr: &Multiaddr) -> Result<(), ConnectionDenied>;
-            fn handle_pending_outbound_connection(&mut self, connection_id: ConnectionId, maybe_peer: Option<PeerId>, addresses: &[Multiaddr], effective_role: Endpoint) -> Result<Vec<Multiaddr>, ConnectionDenied>;
-        }
-    }
-
-    fn handle_established_inbound_connection(
-        &mut self,
-        connection_id: ConnectionId,
-        peer: PeerId,
-        local_addr: &Multiaddr,
-        remote_addr: &Multiaddr,
-    ) -> Result<THandler<Self>, ConnectionDenied> {
-        Ok(ConnectionHandler::select(
-            dummy::ConnectionHandler,
-            self.managed.handle_established_inbound_connection(
-                connection_id,
-                peer,
-                local_addr,
-                remote_addr,
-            )?,
-        ))
-    }
-
-    #[allow(clippy::needless_question_mark)]
-    fn handle_established_outbound_connection(
-        &mut self,
-        connection_id: ConnectionId,
-        peer: PeerId,
-        addr: &Multiaddr,
-        role_override: Endpoint,
-        port_use: PortUse,
-    ) -> Result<THandler<Self>, ConnectionDenied> {
-        Ok(ConnectionHandler::select(
-            dummy::ConnectionHandler,
-            self.managed.handle_established_outbound_connection(
-                connection_id,
-                peer,
-                addr,
-                role_override,
-                port_use,
-            )?,
-        ))
-    }
-
-    fn on_connection_handler_event(
-        &mut self,
-        peer_id: PeerId,
-        connection_id: ConnectionId,
-        event: THandlerOutEvent<Self>,
-    ) {
-        match event {
-            Either::Left(ev) => libp2p::core::util::unreachable(ev),
-            Either::Right(ev) => {
-                self.managed
-                    .on_connection_handler_event(peer_id, connection_id, ev)
-            }
-        }
-    }
-
-    // hook into these methods to drive behavior
-
-    fn on_swarm_event(&mut self, event: FromSwarm) {
-        self.managed.on_swarm_event(event); // let mDNS handle swarm events
-
-        // handle swarm events to update internal state:
-        match event {
-            FromSwarm::ConnectionEstablished(ConnectionEstablished {
-                peer_id,
-                connection_id,
-                endpoint,
-                ..
-            }) => {
-                let remote_address = match endpoint {
-                    ConnectedPoint::Dialer { address, .. } => address,
-                    ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr,
-                };
-
-                if let Some((ip, port)) = remote_address.try_to_tcp_addr() {
-                    // handle connection established event which is filtered correctly
-                    self.on_connection_established(peer_id, connection_id, ip, port)
-                }
-            }
-            FromSwarm::ConnectionClosed(ConnectionClosed {
-                peer_id,
-                connection_id,
-                endpoint,
-                ..
-            }) => {
-                let remote_address = match endpoint {
-                    ConnectedPoint::Dialer { address, .. } => address,
-                    ConnectedPoint::Listener { send_back_addr, .. } => send_back_addr,
-                };
-
-                if let Some((ip, port)) = remote_address.try_to_tcp_addr() {
-                    // handle connection closed event which is filtered correctly
-                    self.on_connection_closed(peer_id, connection_id, ip, port)
-                }
-            }
-
-            // since we are running TCP/IP transport layer, we are assuming that
-            // no address changes can occur, hence encountering one is a fatal error
-            FromSwarm::AddressChange(a) => {
-                unreachable!("unhandlable: address change encountered: {:?}", a)
-            }
-            _ => {}
-        }
-    }
-
-    fn poll(&mut self, cx: &mut Context) -> Poll<ToSwarm<Self::ToSwarm, THandlerInEvent<Self>>> {
-        // delegate to managed behaviors for any behaviors they need to perform
-        match self.managed.poll(cx) {
-            Poll::Ready(ToSwarm::GenerateEvent(e)) => {
-                match e {
-                    // handle discovered and expired events from mDNS
-                    managed::BehaviourEvent::Mdns(e) => match e.clone() {
-                        mdns::Event::Discovered(peers) => {
-                            self.handle_mdns_discovered(peers);
-                        }
-                        mdns::Event::Expired(peers) => {
-                            self.handle_mdns_expired(peers);
-                        }
-                    },
-
-                    // handle ping events => if error then disconnect
-                    managed::BehaviourEvent::Ping(e) => {
-                        if let Err(_) = e.result {
-                            self.close_connection(e.peer, e.connection.clone())
-                        }
-                    }
-                }
-
-                // since we just consumed an event, we should immediately wake just in case
-                // there are more events to come where that came from
-                cx.waker().wake_by_ref();
-            }
-
-            // forward any other mDNS event to the swarm or its connection handler(s)
-            Poll::Ready(e) => {
-                return Poll::Ready(
-                    e.map_out(|_| unreachable!("events returning to swarm already handled"))
-                        .map_in(Either::Right),
-                );
-            }
-
-            Poll::Pending => {}
-        }
-
-        // retry connecting to all mDNS peers periodically (fails safely if already connected)
-        if self.retry_delay.poll_unpin(cx).is_ready() {
-            for (p, mas) in self.mdns_discovered.clone() {
-                for ma in mas {
-                    self.dial(p, ma)
-                }
-            }
-            self.retry_delay.reset(RETRY_CONNECT_INTERVAL) // reset timeout
-        }
-
-        // send out any pending events from our own service
-        if let Some(e) = self.pending_events.pop_front(cx) {
-            return Poll::Ready(e.map_in(Either::Left));
-        }
-
-        // wait for pending events
-        Poll::Pending
-    }
-}
@@ -1,44 +0,0 @@
-use delegate::delegate;
-use libp2p::swarm::handler::ConnectionEvent;
-use libp2p::swarm::{ConnectionHandlerEvent, SubstreamProtocol, dummy, handler};
-use std::task::{Context, Poll};
-
-/// An implementation of [`ConnectionHandler`] that doesn't handle any protocols, but it keeps
-/// the connection alive.
-#[derive(Clone)]
-#[repr(transparent)]
-pub struct ConnectionHandler(dummy::ConnectionHandler);
-
-impl ConnectionHandler {
-    pub fn new() -> Self {
-        ConnectionHandler(dummy::ConnectionHandler)
-    }
-}
-
-impl handler::ConnectionHandler for ConnectionHandler {
-    // delegate types and implementation mostly to dummy handler
-    type FromBehaviour = <dummy::ConnectionHandler as handler::ConnectionHandler>::FromBehaviour;
-    type ToBehaviour = <dummy::ConnectionHandler as handler::ConnectionHandler>::ToBehaviour;
-    type InboundProtocol =
-        <dummy::ConnectionHandler as handler::ConnectionHandler>::InboundProtocol;
-    type OutboundProtocol =
-        <dummy::ConnectionHandler as handler::ConnectionHandler>::OutboundProtocol;
-    type InboundOpenInfo =
-        <dummy::ConnectionHandler as handler::ConnectionHandler>::InboundOpenInfo;
-    type OutboundOpenInfo =
-        <dummy::ConnectionHandler as handler::ConnectionHandler>::OutboundOpenInfo;
-
-    delegate! {
-        to self.0 {
-            fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol, Self::InboundOpenInfo>;
-            fn poll(&mut self, cx: &mut Context<'_>) -> Poll<ConnectionHandlerEvent<Self::OutboundProtocol, Self::OutboundOpenInfo, Self::ToBehaviour>>;
-            fn on_behaviour_event(&mut self, event: Self::FromBehaviour);
-            fn on_connection_event(&mut self, event: ConnectionEvent<Self::InboundProtocol, Self::OutboundProtocol, Self::InboundOpenInfo, Self::OutboundOpenInfo>);
-        }
-    }
-
-    // specifically override this to force connection to stay alive
-    fn connection_keep_alive(&self) -> bool {
-        true
-    }
-}
@@ -1,64 +1,299 @@
-//! TODO: crate documentation
-//!
-//! this is here as a placeholder documentation
-//!
-//!
+use libp2p::{
+    Multiaddr, PeerId,
+    futures::StreamExt,
+    gossipsub::{self, TopicHash},
+    identify,
+    identity::Keypair,
+    mdns,
+    swarm::{NetworkBehaviour, SwarmEvent, dial_opts::DialOpts},
+};
+use std::collections::HashMap;
+use tokio::sync::mpsc;

-// enable Rust-unstable features for convenience
-#![feature(trait_alias)]
-// #![feature(stmt_expr_attributes)]
-// #![feature(unboxed_closures)]
-// #![feature(assert_matches)]
-// #![feature(async_fn_in_dyn_trait)]
-// #![feature(async_for_loop)]
-// #![feature(auto_traits)]
-// #![feature(negative_impls)]
+#[derive(Debug)]
+pub struct ListenError;

-pub mod discovery;
-pub mod keep_alive;
-pub mod swarm;
-
-/// Namespace for all the type/trait aliases used by this crate.
-pub(crate) mod alias {
-    use std::error::Error;
-
-    pub type AnyError = Box<dyn Error + Send + Sync + 'static>;
-    pub type AnyResult<T> = Result<T, AnyError>;
+pub enum FromSwarm {
+    PublishError(gossipsub::PublishError),
+    Discovered(PeerId),
+    Expired(PeerId),
+    Message(PeerId, String, Vec<u8>),
+}
+pub enum ToSwarm {
+    Message(String, Vec<u8>),
+    Subscribe(String),
+    Unsubscribe(String),
 }

-/// Namespace for crate-wide extension traits/methods
-pub(crate) mod ext {
-    use extend::ext;
-    use libp2p::Multiaddr;
-    use libp2p::multiaddr::Protocol;
-    use std::net::IpAddr;
+pub struct Peer {
+    pub swarm: libp2p::Swarm<Behaviour>,
+    to_client: mpsc::Sender<FromSwarm>,
+    from_client: mpsc::Receiver<ToSwarm>,
+    namespace: String,
+    known_peers: HashMap<PeerId, Vec<Multiaddr>>,
+}
+impl Peer {
+    pub fn new(
+        namespace: String,
+        kp: Keypair,
+        to_client: mpsc::Sender<FromSwarm>,
+        from_client: mpsc::Receiver<ToSwarm>,
+    ) -> Result<Self, ListenError> {
+        let mut swarm = libp2p::SwarmBuilder::with_existing_identity(kp)
+            .with_tokio()
+            .with_quic()
+            // TODO(evan) .with_bandwidth_metrics()
+            .with_behaviour(|kp| Behaviour::new(namespace.clone(), kp))
+            .expect("invalid swarm behaviour")
+            .build();

-    #[ext(pub, name = MultiaddrExt)]
-    impl Multiaddr {
-        /// If the multiaddress corresponds to a TCP address, extracts it
-        fn try_to_tcp_addr(&self) -> Option<(IpAddr, u16)> {
-            let mut ps = self.into_iter();
-            let ip = if let Some(p) = ps.next() {
-                match p {
-                    Protocol::Ip4(ip) => IpAddr::V4(ip),
-                    Protocol::Ip6(ip) => IpAddr::V6(ip),
-                    _ => return None,
+        swarm
+            .listen_on("/ip6/::/udp/0/quic-v1".parse().expect("invalid multiaddr"))
+            .map_err(|_| ListenError)?;
+        swarm
+            .listen_on(
+                "/ip4/0.0.0.0/udp/0/quic-v1"
+                    .parse()
+                    .expect("invalid multiaddr"),
+            )
+            .map_err(|_| ListenError)?;
+        Ok(Self {
+            swarm,
+            to_client,
+            from_client,
+            namespace,
+            known_peers: HashMap::default(),
+        })
+    }
+    pub async fn run(&mut self) -> Result<(), ()> {
+        loop {
+            tokio::select! {
+                event = self.swarm.next() => self.handle_event(event.ok_or(())?).await?,
+                msg = self.from_client.recv() => self.handle_message(msg.ok_or(())?).await?,
+            }
+        }
+    }
+    async fn handle_message(&mut self, message: ToSwarm) -> Result<(), ()> {
+        match message {
+            ToSwarm::Message(topic, data) => {
+                if let Err(e) = self
+                    .swarm
+                    .behaviour_mut()
+                    .gossipsub
+                    .publish(TopicHash::from_raw(topic), data)
+                {
+                    self.to_client
+                        .send(FromSwarm::PublishError(e))
+                        .await
+                        .map_err(|_| ())?;
                }
-            } else {
-                return None;
-            };
-            let Some(Protocol::Tcp(port)) = ps.next() else {
-                return None;
-            };
-            Some((ip, port))
+            }
+            ToSwarm::Subscribe(topic) => {
+                match self
+                    .swarm
+                    .behaviour_mut()
+                    .gossipsub
+                    .subscribe(&gossipsub::IdentTopic::new(topic))
+                {
+                    Ok(_) => {}
+                    Err(gossipsub::SubscriptionError::NotAllowed) => {
+                        unreachable!("subscription filter hit")
+                    }
+                    Err(gossipsub::SubscriptionError::PublishError(e)) => self
+                        .to_client
+                        .send(FromSwarm::PublishError(e))
+                        .await
+                        .map_err(|_| ())?,
+                }
+            }
+            ToSwarm::Unsubscribe(topic) => {
+                self.swarm
+                    .behaviour_mut()
+                    .gossipsub
+                    .unsubscribe(&gossipsub::IdentTopic::new(topic));
+            }
+        }
+        Ok(())
+    }
+    async fn handle_event(&mut self, event: SwarmEvent<BehaviourEvent>) -> Result<(), ()> {
+        let SwarmEvent::Behaviour(event) = event else {
+            return Ok(());
+        };
+        match event {
+            BehaviourEvent::Gossipsub(gossipsub::Event::Message { message, .. }) => {
+                if let Some(source) = message.source {
+                    self.to_client
+                        .send(FromSwarm::Message(
+                            source,
+                            message.topic.into_string(),
+                            message.data,
+                        ))
+                        .await
+                        .map_err(|_| ())?;
+                }
+            }
+            BehaviourEvent::Identify(identify::Event::Received { peer_id, info, .. }) => {
+                log::debug!(
+                    "identify from {peer_id}: protocol_version='{}' agent_version='{}' (local namespace='{}')",
+                    info.protocol_version,
+                    info.agent_version,
+                    self.namespace
+                );
+                if info.protocol_version == self.namespace {
+                    self.passed_namespace(peer_id);
+                    self.to_client
+                        .send(FromSwarm::Discovered(peer_id))
+                        .await
+                        .map_err(|_| ())?;
+                } else {
+                    self.failed_namespace(peer_id);
+                }
+            }
+            BehaviourEvent::Mdns(mdns::Event::Discovered(v)) => {
+                for (peer_id, addr) in v {
+                    self.known_peers.entry(peer_id).or_default().push(addr);
+                }
+                for (peer_id, addrs) in &self.known_peers {
+                    // dialopts handles rate limiting, we should check errors if we want to blacklist earlier
+                    let _ = self
+                        .swarm
+                        .dial(DialOpts::peer_id(*peer_id).addresses(addrs.clone()).build());
+                }
+            }
+            BehaviourEvent::Mdns(mdns::Event::Expired(v)) => {
+                for (peer_id, addr) in v {
+                    let addrs = self.known_peers.entry(peer_id).or_default();
+                    addrs.retain(|a| *a != addr);
+                    if addrs.is_empty() {
+                        self.known_peers.remove(&peer_id);
+                        self.swarm
+                            .behaviour_mut()
+                            .gossipsub
+                            .remove_explicit_peer(&peer_id);
+                        self.to_client
+                            .send(FromSwarm::Expired(peer_id))
+                            .await
+                            .map_err(|_| ())?;
+                    }
+                }
+            }
+            _ => {}
+        }
+        Ok(())
+    }
+    fn passed_namespace(&mut self, peer_id: PeerId) {
+        self.swarm
+            .behaviour_mut()
+            .gossipsub
+            .remove_blacklisted_peer(&peer_id);
+        self.swarm
+            .behaviour_mut()
+            .gossipsub
+            .add_explicit_peer(&peer_id);
+    }
+    fn failed_namespace(&mut self, peer_id: PeerId) {
+        self.swarm
+            .behaviour_mut()
+            .gossipsub
+            .blacklist_peer(&peer_id);
+        self.swarm
+            .behaviour_mut()
+            .gossipsub
+            .remove_explicit_peer(&peer_id);
+    }
+}
+
+#[derive(NetworkBehaviour)]
+pub struct Behaviour {
+    gossipsub: gossipsub::Behaviour,
+    mdns: mdns::tokio::Behaviour,
+    identify: identify::Behaviour,
+}
+
+impl Behaviour {
+    fn new(namespace: String, kp: &Keypair) -> Self {
+        let mdns = mdns::Behaviour::new(mdns::Config::default(), kp.public().to_peer_id())
+            .expect("mdns behaviour failed to build");
+
+        let identify =
+            identify::Behaviour::new(identify::Config::new_with_signed_peer_record(namespace, kp));
+
+        let gossipsub = gossipsub::Behaviour::new(
+            gossipsub::MessageAuthenticity::Signed(kp.clone()),
+            gossipsub::ConfigBuilder::default()
+                .max_transmit_size(1024 * 1024)
+                .validation_mode(gossipsub::ValidationMode::Strict)
+                .build()
+                .expect("invalid gossipsub configuration"),
+        )
+        .expect("gossipsub behaviour failed ot build");
+
+        Self {
+            gossipsub,
+            mdns,
+            identify,
        }
    }
 }

-pub(crate) mod private {
-    #![allow(dead_code)]
+// TODO: more tests
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tokio::time::{Duration, timeout};

-    /// Sealed traits support
-    pub trait Sealed {}
-    impl<T: ?Sized> Sealed for T {}
+    fn make_peer(namespace: &str) -> (Peer, mpsc::Receiver<FromSwarm>, mpsc::Sender<ToSwarm>) {
+        let kp = Keypair::generate_ed25519();
+
+        let (to_client_tx, to_client_rx) = mpsc::channel(64);
+        let (to_peer_tx, to_peer_rx) = mpsc::channel(64);
+
+        let peer = Peer::new(namespace.to_string(), kp, to_client_tx, to_peer_rx)
+            .expect("Peer::new should succeed in tests");
+
+        (peer, to_client_rx, to_peer_tx)
+    }
+
+    async fn next_listen_addr(peer: &mut Peer) -> Multiaddr {
+        loop {
+            match peer.swarm.next().await {
+                Some(SwarmEvent::NewListenAddr { address, .. }) => return address,
+                Some(_) => {}
+                None => panic!("swarm stream ended unexpectedly"),
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn subscribe_and_unsubscribe_do_not_error() {
+        let (mut peer, mut events_rx, commands_tx) = make_peer("ns-test");
+
+        // Drive the swarm just enough to get at least one listen address event,
+        // so the background run loop has something initialized.
+        let _addr = next_listen_addr(&mut peer).await;
+
+        // Run the peer loop in the background.
+        let handle = tokio::spawn(async move {
+            let _ = peer.run().await;
+        });
+
+        commands_tx
+            .send(ToSwarm::Subscribe("topic-a".to_string()))
+            .await
+            .unwrap();
+
+        commands_tx
+            .send(ToSwarm::Unsubscribe("topic-a".to_string()))
+            .await
+            .unwrap();
+
+        // We don't *require* any FromSwarm events here; this is mainly a
+        // smoke test that the message-handling path doesn't panic/hang.
+        // Still, poll briefly to ensure the task is alive.
+        let _ = timeout(Duration::from_millis(200), events_rx.recv()).await;
+
+        // Shut down: dropping the command sender closes the channel, causing run() to return Err.
+        drop(commands_tx);
+        let _ = handle.await;
+    }
 }
@@ -1,143 +0,0 @@
-use crate::alias;
-use crate::swarm::transport::tcp_transport;
-pub use behaviour::{Behaviour, BehaviourEvent};
-use libp2p::{SwarmBuilder, identity};
-
-pub type Swarm = libp2p::Swarm<Behaviour>;
-
-/// The current version of the network: this prevents devices running different versions of the
-/// software from interacting with each other.
-///
-/// TODO: right now this is a hardcoded constant; figure out what the versioning semantics should
-///       even be, and how to inject the right version into this config/initialization. E.g. should
-///       this be passed in as a parameter? What about rapidly changing versions in debug builds?
-///       this is all VERY very hard to figure out and needs to be mulled over as a team.
-pub const NETWORK_VERSION: &[u8] = b"v0.0.1";
-pub const OVERRIDE_VERSION_ENV_VAR: &str = "EXO_LIBP2P_NAMESPACE";
-
-/// Create and configure a swarm which listens to all ports on OS
-pub fn create_swarm(keypair: identity::Keypair) -> alias::AnyResult<Swarm> {
-    let mut swarm = SwarmBuilder::with_existing_identity(keypair)
-        .with_tokio()
-        .with_other_transport(tcp_transport)?
-        .with_behaviour(Behaviour::new)?
-        .build();
-
-    // Listen on all interfaces and whatever port the OS assigns
-    swarm.listen_on("/ip4/0.0.0.0/tcp/0".parse()?)?;
-    Ok(swarm)
-}
-
-mod transport {
-    use crate::alias;
-    use crate::swarm::{NETWORK_VERSION, OVERRIDE_VERSION_ENV_VAR};
-    use futures::{AsyncRead, AsyncWrite};
-    use keccak_const::Sha3_256;
-    use libp2p::core::muxing;
-    use libp2p::core::transport::Boxed;
-    use libp2p::pnet::{PnetError, PnetOutput};
-    use libp2p::{PeerId, Transport, identity, noise, pnet, yamux};
-    use std::{env, sync::LazyLock};
-
-    /// Key used for networking's private network; parametrized on the [`NETWORK_VERSION`].
-    /// See [`pnet_upgrade`] for more.
-    static PNET_PRESHARED_KEY: LazyLock<[u8; 32]> = LazyLock::new(|| {
-        let builder = Sha3_256::new().update(b"exo_discovery_network");
-
-        if let Ok(var) = env::var(OVERRIDE_VERSION_ENV_VAR) {
-            let bytes = var.into_bytes();
-            builder.update(&bytes)
-        } else {
-            builder.update(NETWORK_VERSION)
-        }
-        .finalize()
-    });
-
-    /// Make the Swarm run on a private network, as to not clash with public libp2p nodes and
-    /// also different-versioned instances of this same network.
-    /// This is implemented as an additional "upgrade" ontop of existing [`libp2p::Transport`] layers.
-    async fn pnet_upgrade<TSocket>(
-        socket: TSocket,
-        _: impl Sized,
-    ) -> Result<PnetOutput<TSocket>, PnetError>
-    where
-        TSocket: AsyncRead + AsyncWrite + Send + Unpin + 'static,
-    {
-        use pnet::{PnetConfig, PreSharedKey};
-        PnetConfig::new(PreSharedKey::new(*PNET_PRESHARED_KEY))
-            .handshake(socket)
-            .await
-    }
-
-    /// TCP/IP transport layer configuration.
-    pub fn tcp_transport(
-        keypair: &identity::Keypair,
-    ) -> alias::AnyResult<Boxed<(PeerId, muxing::StreamMuxerBox)>> {
-        use libp2p::{
-            core::upgrade::Version,
-            tcp::{Config, tokio},
-        };
-
-        // `TCP_NODELAY` enabled => avoid latency
-        let tcp_config = Config::default().nodelay(true);
-
-        // V1 + lazy flushing => 0-RTT negotiation
-        let upgrade_version = Version::V1Lazy;
-
-        // Noise is faster than TLS + we don't care much for security
-        let noise_config = noise::Config::new(keypair)?;
-
-        // Use default Yamux config for multiplexing
-        let yamux_config = yamux::Config::default();
-
-        // Create new Tokio-driven TCP/IP transport layer
-        let base_transport = tokio::Transport::new(tcp_config)
-            .and_then(pnet_upgrade)
-            .upgrade(upgrade_version)
-            .authenticate(noise_config)
-            .multiplex(yamux_config);
-
-        // Return boxed transport (to flatten complex type)
-        Ok(base_transport.boxed())
-    }
-}
-
-mod behaviour {
-    use crate::{alias, discovery};
-    use libp2p::swarm::NetworkBehaviour;
-    use libp2p::{gossipsub, identity};
-
-    /// Behavior of the Swarm which composes all desired behaviors:
-    /// Right now its just [`discovery::Behaviour`] and [`gossipsub::Behaviour`].
-    #[derive(NetworkBehaviour)]
-    pub struct Behaviour {
-        pub discovery: discovery::Behaviour,
-        pub gossipsub: gossipsub::Behaviour,
-    }
-
-    impl Behaviour {
-        pub fn new(keypair: &identity::Keypair) -> alias::AnyResult<Self> {
-            Ok(Self {
-                discovery: discovery::Behaviour::new(keypair)?,
-                gossipsub: gossipsub_behaviour(keypair),
-            })
-        }
-    }
-
-    fn gossipsub_behaviour(keypair: &identity::Keypair) -> gossipsub::Behaviour {
-        use gossipsub::{ConfigBuilder, MessageAuthenticity, ValidationMode};
-
-        // build a gossipsub network behaviour
-        //  => signed message authenticity + strict validation mode means the message-ID is
-        //     automatically provided by gossipsub w/out needing to provide custom message-ID function
-        gossipsub::Behaviour::new(
-            MessageAuthenticity::Signed(keypair.clone()),
-            ConfigBuilder::default()
-                .max_transmit_size(1024 * 1024)
-                .validation_mode(ValidationMode::Strict)
-                .build()
-                .expect("the configuration should always be valid"),
-        )
-        .expect("creating gossipsub behavior should always work")
-    }
-}
@@ -1,7 +0,0 @@
-// maybe this will hold test in the future...??
-
-#[cfg(test)]
-mod tests {
-    #[test]
-    fn does_nothing() {}
-}
@@ -1,2 +0,0 @@
-[toolchain]
-channel = "nightly"
@@ -1,15 +0,0 @@
-[package]
-name = "util"
-version = { workspace = true }
-edition = { workspace = true }
-publish = false
-
-[lib]
-doctest = false
-name = "util"
-path = "src/lib.rs"
-
-[lints]
-workspace = true
-
-[dependencies]
@@ -1 +0,0 @@
-pub mod wakerdeque;
@@ -1,55 +0,0 @@
-use std::collections::VecDeque;
-use std::fmt::{Debug, Formatter};
-use std::task::{Context, Waker};
-
-/// A wrapper around [`VecDeque`] which wakes (if it can) on any `push_*` methods,
-/// and updates the internally stored waker by consuming [`Context`] on any `pop_*` methods.
-pub struct WakerDeque<T> {
-    waker: Option<Waker>,
-    deque: VecDeque<T>,
-}
-
-impl<T: Debug> Debug for WakerDeque<T> {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        self.deque.fmt(f)
-    }
-}
-
-impl<T> WakerDeque<T> {
-    pub fn new() -> Self {
-        Self {
-            waker: None,
-            deque: VecDeque::new(),
-        }
-    }
-
-    fn update(&mut self, cx: &mut Context<'_>) {
-        self.waker = Some(cx.waker().clone());
-    }
-
-    fn wake(&mut self) {
-        let Some(ref mut w) = self.waker else { return };
-        w.wake_by_ref();
-        self.waker = None;
-    }
-
-    pub fn pop_front(&mut self, cx: &mut Context<'_>) -> Option<T> {
-        self.update(cx);
-        self.deque.pop_front()
-    }
-
-    pub fn pop_back(&mut self, cx: &mut Context<'_>) -> Option<T> {
-        self.update(cx);
-        self.deque.pop_back()
-    }
-
-    pub fn push_front(&mut self, value: T) {
-        self.wake();
-        self.deque.push_front(value);
-    }
-
-    pub fn push_back(&mut self, value: T) {
-        self.wake();
-        self.deque.push_back(value);
-    }
-}
@@ -1,4 +1,5 @@
 import argparse
+import importlib.metadata
 import itertools
 import multiprocessing as mp
 import os
@@ -44,9 +45,9 @@ class Node:
    @classmethod
    async def create(cls, args: "Args") -> "Self":
        keypair = get_node_id_keypair()
-        node_id = NodeId(keypair.to_peer_id().to_base58())
+        node_id = NodeId(keypair.to_string())
        session_id = SessionId(master_node_id=node_id, election_clock=0)
-        router = Router.create(keypair)
+        router = Router.create(keypair, namespace=args.namespace)
        await router.register_topic(topics.GLOBAL_EVENTS)
        await router.register_topic(topics.LOCAL_EVENTS)
        await router.register_topic(topics.COMMANDS)
@@ -72,7 +73,7 @@ class Node:
        else:
            download_coordinator = None

-        if args.spawn_api:
+        if not args.no_api:
            api = API(
                node_id,
                session_id,
@@ -258,7 +259,7 @@ def main():
    # TODO: Refactor the current verbosity system
    logger_setup(EXO_LOG, args.verbosity)
    logger.info("Starting EXO")
-    logger.info(f"EXO_LIBP2P_NAMESPACE: {os.getenv('EXO_LIBP2P_NAMESPACE')}")
+    logger.info(f"Namespace: {args.namespace}")

    # Set FAST_SYNCH override env var for runner subprocesses
    if args.fast_synch is True:
@@ -275,13 +276,13 @@ def main():


 class Args(CamelCaseModel):
-    verbosity: int = 0
-    force_master: bool = False
-    spawn_api: bool = False
-    api_port: PositiveInt = 52415
-    tb_only: bool = False
+    verbosity: int
+    force_master: bool
+    no_api: bool
+    api_port: PositiveInt
    no_worker: bool = False
    no_downloads: bool = False
+    namespace: str
    fast_synch: bool | None = None  # None = auto, True = force on, False = force off

    @classmethod
@@ -311,14 +312,15 @@ class Args(CamelCaseModel):
        )
        parser.add_argument(
            "--no-api",
-            action="store_false",
-            dest="spawn_api",
+            action="store_true",
+            help="Disable the API server for this node",
        )
        parser.add_argument(
            "--api-port",
            type=int,
            dest="api_port",
            default=52415,
+            help="Which port the API server will be available on",
        )
        parser.add_argument(
            "--no-worker",
@@ -329,6 +331,11 @@ class Args(CamelCaseModel):
            action="store_true",
            help="Disable the download coordinator (node won't download models)",
        )
+        parser.add_argument(
+            "--namespace",
+            default=importlib.metadata.version("exo"),
+            help="Set the EXO namespace to run multiple isolated clusters",
+        )
        fast_synch_group = parser.add_mutually_exclusive_group()
        fast_synch_group.add_argument(
            "--fast-synch",
@@ -144,8 +144,8 @@ async def collect_responses_response(
            for tool in chunk.tool_calls:
                function_call_items.append(
                    ResponseFunctionCallItem(
-                        id=f"fc_{tool.id}",
-                        call_id=f"call_{tool.id}",
+                        id=tool.id,
+                        call_id=tool.id,
                        name=tool.name,
                        arguments=tool.arguments,
                    )
@@ -396,7 +396,7 @@ class Master:
                        await self._handle_traces_collected(event)
                        continue

-                    logger.debug(f"Master indexing event: {str(event)[:100]}")
+                    logger.trace(f"Master indexing event: {str(event)[:100]}")
                    indexed = IndexedEvent(event=event, idx=len(self._event_log))
                    self.state = apply(self.state, indexed)

@@ -42,7 +42,7 @@ from exo.utils.channels import channel
@pytest.mark.asyncio
 async def test_master():
    keypair = get_node_id_keypair()
-    node_id = NodeId(keypair.to_peer_id().to_base58())
+    node_id = NodeId(keypair.to_string())
    session_id = SessionId(master_node_id=node_id, election_clock=0)

    ge_sender, global_event_receiver = channel[ForwarderEvent]()
@@ -75,7 +75,7 @@ async def test_master():
    async with anyio.create_task_group() as tg:
        tg.start_soon(master.run)

-        sender_node_id = NodeId(f"{keypair.to_peer_id().to_base58()}_sender")
+        sender_node_id = NodeId(f"{keypair.to_string()}_sender")
        # inject a NodeGatheredInfo event
        logger.info("inject a NodeGatheredInfo event")
        await local_event_sender.send(
@@ -1,37 +1,9 @@
-from enum import Enum
-
-from exo_pyo3_bindings import ConnectionUpdate, ConnectionUpdateType
-
 from exo.shared.types.common import NodeId
 from exo.utils.pydantic_ext import CamelCaseModel

 """Serialisable types for Connection Updates/Messages"""


-class ConnectionMessageType(Enum):
-    Connected = 0
-    Disconnected = 1
-
-    @staticmethod
-    def from_update_type(update_type: ConnectionUpdateType):
-        match update_type:
-            case ConnectionUpdateType.Connected:
-                return ConnectionMessageType.Connected
-            case ConnectionUpdateType.Disconnected:
-                return ConnectionMessageType.Disconnected
-
-
 class ConnectionMessage(CamelCaseModel):
    node_id: NodeId
-    connection_type: ConnectionMessageType
-    remote_ipv4: str
-    remote_tcp_port: int
-
-    @classmethod
-    def from_update(cls, update: ConnectionUpdate) -> "ConnectionMessage":
-        return cls(
-            node_id=NodeId(update.peer_id.to_base58()),
-            connection_type=ConnectionMessageType.from_update_type(update.update_type),
-            remote_ipv4=update.remote_ipv4,
-            remote_tcp_port=update.remote_tcp_port,
-        )
+    expired: bool
@@ -1,5 +1,5 @@
 from copy import copy
-from itertools import count
+from dataclasses import dataclass, field
 from math import inf
 from os import PathLike
 from pathlib import Path
@@ -14,15 +14,14 @@ from anyio import (
 )
 from anyio.abc import TaskGroup
 from exo_pyo3_bindings import (
-    AllQueuesFullError,
    Keypair,
-    NetworkingHandle,
-    NoPeersSubscribedToTopicError,
+    PyPeer,
 )
 from filelock import FileLock
 from loguru import logger

 from exo.shared.constants import EXO_NODE_ID_KEYPAIR
+from exo.shared.types.common import NodeId
 from exo.utils.channels import Receiver, Sender, channel
 from exo.utils.pydantic_ext import CamelCaseModel

@@ -99,28 +98,32 @@ class TopicRouter[T: CamelCaseModel]:
        )


+@dataclass
 class Router:
-    @classmethod
-    def create(cls, identity: Keypair) -> "Router":
-        return cls(handle=NetworkingHandle(identity))
+    _peer: PyPeer
+    topic_routers: dict[str, TopicRouter[CamelCaseModel]] = field(
+        init=False, default_factory=dict
+    )
+    networking_receiver: Receiver[tuple[str, bytes]] = field(init=False)
+    _tmp_networking_sender: Sender[tuple[str, bytes]] | None = field(init=False)
+    _tg: TaskGroup | None = None

-    def __init__(self, handle: NetworkingHandle):
-        self.topic_routers: dict[str, TopicRouter[CamelCaseModel]] = {}
-        send, recv = channel[tuple[str, bytes]]()
-        self.networking_receiver: Receiver[tuple[str, bytes]] = recv
-        self._net: NetworkingHandle = handle
-        self._tmp_networking_sender: Sender[tuple[str, bytes]] | None = send
-        self._id_count = count()
-        self._tg: TaskGroup | None = None
+    def __post_init__(self):
+        self._tmp_networking_sender, self.networking_receiver = channel()
+
+    @classmethod
+    def create(cls, identity: Keypair, namespace: str) -> "Router":
+        return cls(_peer=PyPeer.new(identity, namespace))

    async def register_topic[T: CamelCaseModel](self, topic: TypedTopic[T]):
-        assert self._tg is None, "Attempted to register topic after setup time"
        send = self._tmp_networking_sender
        if send:
            self._tmp_networking_sender = None
        else:
            send = self.networking_receiver.clone_sender()
        router = TopicRouter[T](topic, send)
+        if self._tg is not None:
+            self._tg.start_soon(router.run)
        self.topic_routers[topic.topic] = cast(TopicRouter[CamelCaseModel], router)
        await self._networking_subscribe(str(topic.topic))

@@ -148,14 +151,18 @@ class Router:
    async def run(self):
        logger.debug("Starting Router")
        try:
+
+            async def _peer_run():
+                await self._peer.run()
+
            async with create_task_group() as tg:
                self._tg = tg
                for topic in self.topic_routers:
                    router = self.topic_routers[topic]
                    tg.start_soon(router.run)
                tg.start_soon(self._networking_recv)
-                tg.start_soon(self._networking_recv_connection_messages)
                tg.start_soon(self._networking_publish)
+                tg.start_soon(_peer_run)
                # Router only shuts down if you cancel it.
                await sleep_forever()
        finally:
@@ -170,47 +177,58 @@ class Router:
        self._tg.cancel_scope.cancel()

    async def _networking_subscribe(self, topic: str):
-        await self._net.gossipsub_subscribe(topic)
+        await self._peer.subscribe(topic)
        logger.info(f"Subscribed to {topic}")

    async def _networking_unsubscribe(self, topic: str):
-        await self._net.gossipsub_unsubscribe(topic)
+        await self._peer.unsubscribe(topic)
        logger.info(f"Unsubscribed from {topic}")

    async def _networking_recv(self):
        while True:
-            topic, data = await self._net.gossipsub_recv()
-            logger.trace(f"Received message on {topic} with payload {data}")
+            try:
+                swarm_event = await self._peer.recv()
+            except ValueError:
+                logger.error("Message too large for gossipsub, dropped")
+                continue
+            except ConnectionError:
+                logger.error("All peer queues full, network overloaded")
+                continue
+            except RuntimeError:
+                break
+
+            cm = None
+            if (peer_id := swarm_event.downcast_discovered()) is not None:
+                cm = ConnectionMessage(node_id=NodeId(peer_id), expired=False)
+            if (peer_id := swarm_event.downcast_expired()) is not None:
+                cm = ConnectionMessage(node_id=NodeId(peer_id), expired=True)
+
+            if cm is not None:
+                if CONNECTION_MESSAGES.topic in self.topic_routers:
+                    router = self.topic_routers[CONNECTION_MESSAGES.topic]
+                    assert router.topic.model_type == ConnectionMessage
+                    router = cast(TopicRouter[ConnectionMessage], router)
+                    await router.publish(cm)
+                continue
+
+            assert (msg := swarm_event.downcast_message()) is not None
+            _origin, topic, payload = msg
+            logger.debug(f"Received message on {topic} with payload {payload}")
            if topic not in self.topic_routers:
                logger.warning(f"Received message on unknown or inactive topic {topic}")
                continue

            router = self.topic_routers[topic]
-            await router.publish_bytes(data)
-
-    async def _networking_recv_connection_messages(self):
-        while True:
-            update = await self._net.connection_update_recv()
-            message = ConnectionMessage.from_update(update)
-            logger.trace(
-                f"Received message on connection_messages with payload {message}"
-            )
-            if CONNECTION_MESSAGES.topic in self.topic_routers:
-                router = self.topic_routers[CONNECTION_MESSAGES.topic]
-                assert router.topic.model_type == ConnectionMessage
-                router = cast(TopicRouter[ConnectionMessage], router)
-                await router.publish(message)
+            await router.publish_bytes(payload)

    async def _networking_publish(self):
        with self.networking_receiver as networked_items:
            async for topic, data in networked_items:
                try:
                    logger.trace(f"Sending message on {topic} with payload {data}")
-                    await self._net.gossipsub_publish(topic, data)
-                except NoPeersSubscribedToTopicError:
-                    pass
-                except AllQueuesFullError:
-                    logger.warning(f"All peer queues full, dropping message on {topic}")
+                    await self._peer.send(topic, data)
+                except RuntimeError:
+                    break


 def get_node_id_keypair(
@@ -221,7 +239,7 @@ def get_node_id_keypair(
    Obtain the :class:`PeerId` by from it.
    """
    # TODO(evan): bring back node id persistence once we figure out how to deal with duplicates
-    return Keypair.generate_ed25519()
+    return Keypair.generate()

    def lock_path(path: str | bytes | PathLike[str] | PathLike[bytes]) -> Path:
        return Path(str(path) + ".lock")
@@ -1,7 +1,7 @@
 import pytest
 from anyio import create_task_group, fail_after, move_on_after

-from exo.routing.connection_message import ConnectionMessage, ConnectionMessageType
+from exo.routing.connection_message import ConnectionMessage
 from exo.shared.election import Election, ElectionMessage, ElectionResult
 from exo.shared.types.commands import ForwarderCommand, TestCommand
 from exo.shared.types.common import NodeId, SessionId
@@ -330,9 +330,7 @@ async def test_connection_message_triggers_new_round_broadcast() -> None:
            await cm_tx.send(
                ConnectionMessage(
                    node_id=NodeId(),
-                    connection_type=ConnectionMessageType.Connected,
-                    remote_ipv4="",
-                    remote_tcp_port=0,
+                    expired=False,
                )
            )

@@ -1,307 +0,0 @@
-"""Shared fixtures and helpers for E2E chaos/networking tests.
-
-Provides a ``MiniCluster`` that wires Master + Worker(s) + Election together
-using in-process channels.  No Docker, no network, no GPU -- pure async
-integration testing of the coordination layer.
-"""
-
-from collections.abc import Iterator
-from datetime import datetime, timezone
-from typing import Final
-
-import pytest
-from _pytest.logging import LogCaptureFixture
-from loguru import logger
-
-from exo.master.main import Master
-from exo.shared.models.model_cards import ModelCard, ModelTask
-from exo.shared.types.commands import (
-    CommandId,
-    ForwarderCommand,
-    ForwarderDownloadCommand,
-    PlaceInstance,
-    TextGeneration,
-)
-from exo.shared.types.common import ModelId, NodeId, SessionId
-from exo.shared.types.events import (
-    ForwarderEvent,
-    IndexedEvent,
-    NodeGatheredInfo,
-    TopologyEdgeCreated,
-)
-from exo.shared.types.memory import Memory
-from exo.shared.types.multiaddr import Multiaddr
-from exo.shared.types.profiling import MemoryUsage
-from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
-from exo.shared.types.topology import Connection, SocketConnection
-from exo.shared.types.worker.instances import InstanceMeta
-from exo.shared.types.worker.shards import Sharding
-from exo.utils.channels import Receiver, Sender, channel
-from exo.worker.main import Worker
-
-# ---------------------------------------------------------------------------
-# Constants
-# ---------------------------------------------------------------------------
-
-TEST_MODEL_ID: Final[ModelId] = ModelId("test-model/chaos-test-1b")
-TEST_MODEL_CARD: Final[ModelCard] = ModelCard(
-    model_id=TEST_MODEL_ID,
-    n_layers=16,
-    storage_size=Memory.from_bytes(678_948),
-    hidden_size=2048,
-    supports_tensor=True,
-    tasks=[ModelTask.TextGeneration],
-)
-
-FAST_ELECTION_TIMEOUT: Final[float] = 0.1
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def make_node_id(label: str) -> NodeId:
-    return NodeId(f"node-{label}")
-
-
-def make_session_id(master_node: NodeId) -> SessionId:
-    return SessionId(master_node_id=master_node, election_clock=0)
-
-
-def make_memory_info() -> MemoryUsage:
-    return MemoryUsage(
-        ram_total=Memory.from_bytes(16 * 1024 * 1024 * 1024),
-        ram_available=Memory.from_bytes(8 * 1024 * 1024 * 1024),
-        swap_total=Memory.from_bytes(0),
-        swap_available=Memory.from_bytes(0),
-    )
-
-
-def make_gathered_info_event(
-    node_id: NodeId, sender_id: NodeId, session_id: SessionId, origin_idx: int
-) -> ForwarderEvent:
-    return ForwarderEvent(
-        origin_idx=origin_idx,
-        origin=sender_id,
-        session=session_id,
-        event=NodeGatheredInfo(
-            when=str(datetime.now(tz=timezone.utc)),
-            node_id=node_id,
-            info=make_memory_info(),
-        ),
-    )
-
-
-def make_topology_edge_event(
-    source: NodeId,
-    sink: NodeId,
-    sender_id: NodeId,
-    session_id: SessionId,
-    origin_idx: int,
-    ip_suffix: int = 1,
-) -> ForwarderEvent:
-    """Create a ForwarderEvent wrapping a TopologyEdgeCreated event."""
-    return ForwarderEvent(
-        origin_idx=origin_idx,
-        origin=sender_id,
-        session=session_id,
-        event=TopologyEdgeCreated(
-            conn=Connection(
-                source=source,
-                sink=sink,
-                edge=SocketConnection(
-                    sink_multiaddr=Multiaddr(
-                        address=f"/ip4/10.0.0.{ip_suffix}/tcp/52415"
-                    )
-                ),
-            )
-        ),
-    )
-
-
-class EventCollector:
-    """Collects ForwarderEvents from a global event receiver."""
-
-    def __init__(self, receiver: Receiver[ForwarderEvent]) -> None:
-        self._receiver = receiver
-        self.indexed_events: list[IndexedEvent] = []
-
-    def collect(self) -> list[IndexedEvent]:
-        raw = self._receiver.collect()
-        for fe in raw:
-            self.indexed_events.append(
-                IndexedEvent(event=fe.event, idx=len(self.indexed_events))
-            )
-        return self.indexed_events
-
-    async def wait_for_event_count(
-        self, count: int, *, timeout: float = 5.0, poll_interval: float = 0.01
-    ) -> list[IndexedEvent]:
-        import anyio
-
-        with anyio.fail_after(timeout):
-            while len(self.collect()) < count:
-                await anyio.sleep(poll_interval)
-        return self.indexed_events
-
-
-class MiniCluster:
-    """An in-process cluster with one Master and N Workers wired via channels.
-
-    No networking, no real model loading -- exercises the coordination logic
-    (event sourcing, command routing, election) in a deterministic, fast
-    test harness.
-    """
-
-    def __init__(self, node_count: int = 2) -> None:
-        self.node_count = node_count
-        self.master_node_id = make_node_id("master")
-        self.session_id = make_session_id(self.master_node_id)
-
-        # -- shared bus channels --
-        self.global_event_sender: Sender[ForwarderEvent]
-        self.global_event_internal_receiver: Receiver[ForwarderEvent]
-        self.global_event_sender, self.global_event_internal_receiver = channel[
-            ForwarderEvent
-        ]()
-
-        self.command_sender: Sender[ForwarderCommand]
-        self.command_receiver: Receiver[ForwarderCommand]
-        self.command_sender, self.command_receiver = channel[ForwarderCommand]()
-
-        self.local_event_sender: Sender[ForwarderEvent]
-        self.local_event_receiver: Receiver[ForwarderEvent]
-        self.local_event_sender, self.local_event_receiver = channel[ForwarderEvent]()
-
-        self.download_cmd_sender: Sender[ForwarderDownloadCommand]
-        self._download_cmd_receiver: Receiver[ForwarderDownloadCommand]
-        self.download_cmd_sender, self._download_cmd_receiver = channel[
-            ForwarderDownloadCommand
-        ]()
-
-        # -- event collector (taps global events) --
-        self.event_collector = EventCollector(
-            self.global_event_internal_receiver.clone()
-        )
-
-        # -- master --
-        self.master = Master(
-            self.master_node_id,
-            self.session_id,
-            global_event_sender=self.global_event_sender.clone(),
-            local_event_receiver=self.local_event_receiver.clone(),
-            command_receiver=self.command_receiver.clone(),
-            download_command_sender=self.download_cmd_sender.clone(),
-        )
-
-        # -- workers --
-        self.worker_node_ids: list[NodeId] = []
-        self.workers: list[Worker] = []
-        for i in range(node_count):
-            wid = make_node_id(f"worker-{i}")
-            self.worker_node_ids.append(wid)
-
-            counter: Iterator[int] = iter(range(1_000_000))
-            worker = Worker(
-                wid,
-                self.session_id,
-                global_event_receiver=self.global_event_internal_receiver.clone(),
-                local_event_sender=self.local_event_sender.clone(),
-                command_sender=self.command_sender.clone(),
-                download_command_sender=self.download_cmd_sender.clone(),
-                event_index_counter=counter,
-            )
-            self.workers.append(worker)
-
-    async def inject_node_info(self, node_id: NodeId, sender_suffix: str = "") -> None:
-        """Inject a NodeGatheredInfo event for a node into the local event bus."""
-        sender_id = NodeId(f"{node_id}_sender{sender_suffix}")
-        await self.local_event_sender.send(
-            make_gathered_info_event(node_id, sender_id, self.session_id, 0)
-        )
-
-    async def wait_for_topology_nodes(
-        self, count: int, *, timeout: float = 5.0
-    ) -> None:
-        import anyio
-
-        with anyio.fail_after(timeout):
-            while len(list(self.master.state.topology.list_nodes())) < count:
-                await anyio.sleep(0.01)
-
-    async def place_model(
-        self,
-        model_card: ModelCard | None = None,
-        min_nodes: int = 1,
-    ) -> None:
-        card = model_card or TEST_MODEL_CARD
-        await self.command_sender.send(
-            ForwarderCommand(
-                origin=self.master_node_id,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=card,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=min_nodes,
-                ),
-            )
-        )
-
-    async def wait_for_instances(self, count: int, *, timeout: float = 5.0) -> None:
-        import anyio
-
-        with anyio.fail_after(timeout):
-            while len(self.master.state.instances) < count:
-                await anyio.sleep(0.01)
-
-    async def send_chat(
-        self,
-        message: str,
-        model: ModelId | None = None,
-    ) -> CommandId:
-        cmd_id = CommandId()
-        await self.command_sender.send(
-            ForwarderCommand(
-                origin=self.master_node_id,
-                command=TextGeneration(
-                    command_id=cmd_id,
-                    task_params=TextGenerationTaskParams(
-                        model=model or TEST_MODEL_ID,
-                        input=[InputMessage(role="user", content=message)],
-                    ),
-                ),
-            )
-        )
-        return cmd_id
-
-    async def shutdown_master(self) -> None:
-        await self.master.shutdown()
-
-    def shutdown_workers(self) -> None:
-        for w in self.workers:
-            w.shutdown()
-
-
-# ---------------------------------------------------------------------------
-# Fixtures
-# ---------------------------------------------------------------------------
-
-
-@pytest.fixture(autouse=True)
-def fast_election_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
-    monkeypatch.setattr("exo.shared.election.DEFAULT_ELECTION_TIMEOUT", 0.1)
-
-
-@pytest.fixture
-def caplog(caplog: LogCaptureFixture) -> Iterator[LogCaptureFixture]:
-    handler_id = logger.add(
-        caplog.handler,
-        format="{message}",
-        level=0,
-        filter=lambda record: record["level"].no >= caplog.handler.level,
-        enqueue=True,
-    )
-    yield caplog
-    logger.remove(handler_id)
@@ -1,255 +0,0 @@
-"""E2E Chaos Test: Client disconnect.
-
-Scenarios:
-1. Task cancellation after client disconnect -- a TextGeneration command is
-   sent, then immediately cancelled (simulating browser tab close).
-   Verify the master correctly transitions the task to Cancelled status.
-2. Multiple rapid cancellations -- several chat commands are sent and
-   cancelled in quick succession; no tasks should remain in a stuck state.
-"""
-
-import anyio
-import pytest
-
-from exo.master.main import Master
-from exo.shared.types.commands import (
-    CommandId,
-    ForwarderCommand,
-    ForwarderDownloadCommand,
-    PlaceInstance,
-    TaskCancelled,
-    TextGeneration,
-)
-from exo.shared.types.common import NodeId, SessionId
-from exo.shared.types.events import (
-    ForwarderEvent,
-)
-from exo.shared.types.tasks import TaskStatus
-from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
-from exo.shared.types.worker.instances import InstanceMeta
-from exo.shared.types.worker.shards import Sharding
-from exo.utils.channels import channel
-
-from .conftest import (
-    TEST_MODEL_CARD,
-    TEST_MODEL_ID,
-    EventCollector,
-    make_gathered_info_event,
-    make_node_id,
-)
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_task_cancelled_after_client_disconnect() -> None:
-    """Simulate a browser tab close by sending a TextGeneration command
-    followed immediately by a TaskCancelled command.  Verify the task
-    transitions to Cancelled status.
-    """
-    master_nid = make_node_id("master-cancel")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    _collector = EventCollector(ge_receiver.clone())
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register node
-        sender_id = NodeId(f"{master_nid}_sender")
-        await le_sender.send(
-            make_gathered_info_event(master_nid, sender_id, session_id, 0)
-        )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) == 0:
-                await anyio.sleep(0.01)
-
-        # Place instance
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        # Send a chat command
-        chat_cmd_id = CommandId()
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=TextGeneration(
-                    command_id=chat_cmd_id,
-                    task_params=TextGenerationTaskParams(
-                        model=TEST_MODEL_ID,
-                        input=[InputMessage(role="user", content="Hello world")],
-                    ),
-                ),
-            )
-        )
-
-        # Wait for the task to be created
-        with anyio.fail_after(3):
-            while len(master.state.tasks) == 0:
-                await anyio.sleep(0.01)
-
-        # Immediately cancel -- simulating browser tab close
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=TaskCancelled(
-                    command_id=CommandId(),
-                    cancelled_command_id=chat_cmd_id,
-                ),
-            )
-        )
-
-        # Wait for the task status to be updated to Cancelled
-        with anyio.fail_after(3):
-            while True:
-                tasks_cancelled = [
-                    t
-                    for t in master.state.tasks.values()
-                    if t.task_status == TaskStatus.Cancelled
-                ]
-                if tasks_cancelled:
-                    break
-                await anyio.sleep(0.01)
-
-        assert len(tasks_cancelled) == 1
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_rapid_cancel_does_not_leave_stuck_tasks() -> None:
-    """Send multiple chat commands and cancel them all rapidly.
-    Verify no tasks remain in Pending or Running state.
-    """
-    master_nid = make_node_id("master-rapid-cancel")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, _ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register node and place instance
-        sender_id = NodeId(f"{master_nid}_sender")
-        await le_sender.send(
-            make_gathered_info_event(master_nid, sender_id, session_id, 0)
-        )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) == 0:
-                await anyio.sleep(0.01)
-
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        # Send 5 chat commands and immediately cancel each
-        chat_cmd_ids: list[CommandId] = []
-        for i in range(5):
-            cmd_id = CommandId()
-            chat_cmd_ids.append(cmd_id)
-            await cmd_sender.send(
-                ForwarderCommand(
-                    origin=master_nid,
-                    command=TextGeneration(
-                        command_id=cmd_id,
-                        task_params=TextGenerationTaskParams(
-                            model=TEST_MODEL_ID,
-                            input=[InputMessage(role="user", content=f"Message {i}")],
-                        ),
-                    ),
-                )
-            )
-
-        # Wait for all tasks to be created
-        with anyio.fail_after(3):
-            while len(master.state.tasks) < 5:
-                await anyio.sleep(0.01)
-
-        # Cancel all of them
-        for cmd_id in chat_cmd_ids:
-            await cmd_sender.send(
-                ForwarderCommand(
-                    origin=master_nid,
-                    command=TaskCancelled(
-                        command_id=CommandId(),
-                        cancelled_command_id=cmd_id,
-                    ),
-                )
-            )
-
-        # Wait for all cancellations to be processed
-        with anyio.fail_after(3):
-            while True:
-                cancelled_count = sum(
-                    1
-                    for t in master.state.tasks.values()
-                    if t.task_status == TaskStatus.Cancelled
-                )
-                if cancelled_count == 5:
-                    break
-                await anyio.sleep(0.01)
-
-        # No tasks should be Pending or Running
-        stuck = [
-            t
-            for t in master.state.tasks.values()
-            if t.task_status in (TaskStatus.Pending, TaskStatus.Running)
-        ]
-        assert len(stuck) == 0
-
-        await master.shutdown()
@@ -1,395 +0,0 @@
-"""E2E Chaos Test: Concurrent requests.
-
-Scenarios:
-1. Multiple simultaneous inference requests -- verify they are all created
-   as tasks with no data corruption (unique task IDs, correct model IDs).
-2. Concurrent requests across multiple model instances -- verify tasks are
-   routed to the correct instances.
-3. Concurrent requests with load balancing -- when multiple instances of
-   the same model exist, verify tasks are distributed.
-"""
-
-import anyio
-import pytest
-
-from exo.master.main import Master
-from exo.shared.models.model_cards import ModelCard, ModelTask
-from exo.shared.types.commands import (
-    CommandId,
-    ForwarderCommand,
-    ForwarderDownloadCommand,
-    PlaceInstance,
-    TextGeneration,
-)
-from exo.shared.types.common import ModelId, NodeId, SessionId
-from exo.shared.types.events import (
-    ForwarderEvent,
-)
-from exo.shared.types.memory import Memory
-from exo.shared.types.tasks import TaskStatus
-from exo.shared.types.tasks import TextGeneration as TextGenerationTask
-from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
-from exo.shared.types.worker.instances import InstanceMeta
-from exo.shared.types.worker.shards import Sharding
-from exo.utils.channels import channel
-
-from .conftest import (
-    TEST_MODEL_CARD,
-    TEST_MODEL_ID,
-    EventCollector,
-    make_gathered_info_event,
-    make_node_id,
-)
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_concurrent_chat_requests_no_corruption() -> None:
-    """Send multiple TextGeneration commands concurrently and verify each
-    results in a unique task with the correct model and content mapping.
-    """
-    master_nid = make_node_id("master-concurrent")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    _collector = EventCollector(ge_receiver.clone())
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Set up node and instance
-        sender_id = NodeId(f"{master_nid}_sender")
-        await le_sender.send(
-            make_gathered_info_event(master_nid, sender_id, session_id, 0)
-        )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) == 0:
-                await anyio.sleep(0.01)
-
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        # Send 10 concurrent chat requests
-        num_requests = 10
-        cmd_ids: list[CommandId] = []
-
-        async def send_chat(index: int) -> None:
-            cmd_id = CommandId()
-            cmd_ids.append(cmd_id)
-            await cmd_sender.send(
-                ForwarderCommand(
-                    origin=master_nid,
-                    command=TextGeneration(
-                        command_id=cmd_id,
-                        task_params=TextGenerationTaskParams(
-                            model=TEST_MODEL_ID,
-                            input=[
-                                InputMessage(
-                                    role="user",
-                                    content=f"Concurrent request #{index}",
-                                )
-                            ],
-                        ),
-                    ),
-                )
-            )
-
-        async with anyio.create_task_group() as send_tg:
-            for i in range(num_requests):
-                send_tg.start_soon(send_chat, i)
-
-        # Wait for all tasks to be created
-        with anyio.fail_after(5):
-            while len(master.state.tasks) < num_requests:
-                await anyio.sleep(0.01)
-
-        # Verify no corruption
-        assert len(master.state.tasks) == num_requests
-
-        # All task IDs should be unique
-        task_ids = list(master.state.tasks.keys())
-        assert len(set(task_ids)) == num_requests
-
-        # All tasks should target the correct model
-        for task in master.state.tasks.values():
-            assert isinstance(task, TextGenerationTask)
-            assert task.task_params.model == TEST_MODEL_ID
-            assert task.task_status == TaskStatus.Pending
-
-        # All tasks should reference the same instance
-        instance_ids = {task.instance_id for task in master.state.tasks.values()}
-        assert len(instance_ids) == 1
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_concurrent_requests_across_multiple_models() -> None:
-    """Place two different models, then send concurrent requests for each.
-    Verify tasks are routed to the correct model instances.
-    """
-    master_nid = make_node_id("master-multi-model")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, _ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register node
-        sender_id = NodeId(f"{master_nid}_sender")
-        await le_sender.send(
-            make_gathered_info_event(master_nid, sender_id, session_id, 0)
-        )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) == 0:
-                await anyio.sleep(0.01)
-
-        # Place two different models
-        model_a_id = ModelId("test-model/model-a")
-        model_a_card = ModelCard(
-            model_id=model_a_id,
-            n_layers=16,
-            storage_size=Memory.from_bytes(500_000),
-            hidden_size=2048,
-            supports_tensor=True,
-            tasks=[ModelTask.TextGeneration],
-        )
-
-        model_b_id = ModelId("test-model/model-b")
-        model_b_card = ModelCard(
-            model_id=model_b_id,
-            n_layers=32,
-            storage_size=Memory.from_bytes(500_000),
-            hidden_size=4096,
-            supports_tensor=True,
-            tasks=[ModelTask.TextGeneration],
-        )
-
-        for card in [model_a_card, model_b_card]:
-            await cmd_sender.send(
-                ForwarderCommand(
-                    origin=master_nid,
-                    command=PlaceInstance(
-                        command_id=CommandId(),
-                        model_card=card,
-                        sharding=Sharding.Pipeline,
-                        instance_meta=InstanceMeta.MlxRing,
-                        min_nodes=1,
-                    ),
-                )
-            )
-
-        with anyio.fail_after(5):
-            while len(master.state.instances) < 2:
-                await anyio.sleep(0.01)
-
-        # Map instance IDs to models
-        instance_to_model: dict[str, ModelId] = {}
-        for iid, inst in master.state.instances.items():
-            instance_to_model[iid] = inst.shard_assignments.model_id
-
-        # Send concurrent requests for both models
-        async def send_for_model(model_id: ModelId, count: int) -> None:
-            for i in range(count):
-                await cmd_sender.send(
-                    ForwarderCommand(
-                        origin=master_nid,
-                        command=TextGeneration(
-                            command_id=CommandId(),
-                            task_params=TextGenerationTaskParams(
-                                model=model_id,
-                                input=[
-                                    InputMessage(
-                                        role="user",
-                                        content=f"Request for {model_id} #{i}",
-                                    )
-                                ],
-                            ),
-                        ),
-                    )
-                )
-
-        async with anyio.create_task_group() as send_tg:
-            send_tg.start_soon(send_for_model, model_a_id, 3)
-            send_tg.start_soon(send_for_model, model_b_id, 3)
-
-        # Wait for all 6 tasks
-        with anyio.fail_after(5):
-            while len(master.state.tasks) < 6:
-                await anyio.sleep(0.01)
-
-        # Verify task routing
-        model_a_tasks = [
-            t
-            for t in master.state.tasks.values()
-            if isinstance(t, TextGenerationTask) and t.task_params.model == model_a_id
-        ]
-        model_b_tasks = [
-            t
-            for t in master.state.tasks.values()
-            if isinstance(t, TextGenerationTask) and t.task_params.model == model_b_id
-        ]
-
-        assert len(model_a_tasks) == 3
-        assert len(model_b_tasks) == 3
-
-        # All model_a tasks should reference the model_a instance
-        model_a_instance_ids = {
-            iid for iid, mid in instance_to_model.items() if mid == model_a_id
-        }
-        for task in model_a_tasks:
-            assert task.instance_id in model_a_instance_ids
-
-        # All model_b tasks should reference the model_b instance
-        model_b_instance_ids = {
-            iid for iid, mid in instance_to_model.items() if mid == model_b_id
-        }
-        for task in model_b_tasks:
-            assert task.instance_id in model_b_instance_ids
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_event_index_monotonically_increases_under_load() -> None:
-    """Under heavy concurrent command load, verify the master's event log
-    index increases monotonically with no gaps or duplicates.
-    """
-    master_nid = make_node_id("master-monotonic")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    collector = EventCollector(ge_receiver.clone())
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register node and place instance
-        sender_id = NodeId(f"{master_nid}_sender")
-        await le_sender.send(
-            make_gathered_info_event(master_nid, sender_id, session_id, 0)
-        )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) == 0:
-                await anyio.sleep(0.01)
-
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        # Blast 20 concurrent commands
-        async def blast_commands(start: int, count: int) -> None:
-            for i in range(count):
-                await cmd_sender.send(
-                    ForwarderCommand(
-                        origin=master_nid,
-                        command=TextGeneration(
-                            command_id=CommandId(),
-                            task_params=TextGenerationTaskParams(
-                                model=TEST_MODEL_ID,
-                                input=[
-                                    InputMessage(
-                                        role="user",
-                                        content=f"Blast {start + i}",
-                                    )
-                                ],
-                            ),
-                        ),
-                    )
-                )
-
-        async with anyio.create_task_group() as blast_tg:
-            blast_tg.start_soon(blast_commands, 0, 10)
-            blast_tg.start_soon(blast_commands, 10, 10)
-
-        # Wait for all tasks
-        with anyio.fail_after(5):
-            while len(master.state.tasks) < 20:
-                await anyio.sleep(0.01)
-
-        # Collect all events and verify monotonic indexing
-        # NodeGatheredInfo(0) + InstanceCreated(1) + 20 TaskCreated = 22 events
-        await collector.wait_for_event_count(22, timeout=5.0)
-
-        events = collector.indexed_events
-        indices = [e.idx for e in events]
-
-        # Should be 0, 1, 2, ..., N-1 with no gaps
-        expected = list(range(len(indices)))
-        assert indices == expected
-
-        # last_event_applied_idx should match
-        assert master.state.last_event_applied_idx == len(events) - 1
-
-        await master.shutdown()
@@ -1,356 +0,0 @@
-"""E2E Chaos Test: Large model distributed loading.
-
-Scenarios:
-1. Multi-node sharding -- place a model with min_nodes > 1, verify sharding
-   is distributed across multiple nodes with correct shard assignments.
-2. Single-node gets all layers -- place on 1 node, verify full assignment.
-3. Three-node sharding -- verify 3-way distribution.
-"""
-
-import anyio
-import pytest
-
-from exo.master.main import Master
-from exo.shared.models.model_cards import ModelCard, ModelTask
-from exo.shared.types.commands import (
-    CommandId,
-    ForwarderCommand,
-    ForwarderDownloadCommand,
-    PlaceInstance,
-)
-from exo.shared.types.common import ModelId, NodeId, SessionId
-from exo.shared.types.events import (
-    ForwarderEvent,
-)
-from exo.shared.types.memory import Memory
-from exo.shared.types.worker.instances import InstanceMeta, MlxRingInstance
-from exo.shared.types.worker.shards import PipelineShardMetadata, Sharding
-from exo.utils.channels import Sender, channel
-
-from .conftest import (
-    TEST_MODEL_CARD,
-    make_gathered_info_event,
-    make_node_id,
-    make_topology_edge_event,
-)
-
-# A model large enough to need sharding but small enough to fit in test node memory
-# Each test node has 8GB available, so 2 nodes = 16GB, 3 nodes = 24GB.
-# storage_size < total cluster memory to pass the memory filter.
-LARGE_MODEL_CARD = ModelCard(
-    model_id=ModelId("test-model/large-70b-4bit"),
-    n_layers=80,
-    storage_size=Memory.from_bytes(4 * 1024 * 1024 * 1024),
-    hidden_size=8192,
-    supports_tensor=True,
-    tasks=[ModelTask.TextGeneration],
-)
-
-
-async def _register_node(
-    le_sender: Sender[ForwarderEvent],
-    node_id: NodeId,
-    session_id: SessionId,
-) -> None:
-    """Register a node by injecting NodeGatheredInfo."""
-    sender_id = NodeId(f"{node_id}_sender")
-    await le_sender.send(make_gathered_info_event(node_id, sender_id, session_id, 0))
-
-
-async def _add_bidirectional_edge(
-    le_sender: Sender[ForwarderEvent],
-    node_a: NodeId,
-    node_b: NodeId,
-    session_id: SessionId,
-    sender_id: NodeId,
-    origin_idx_start: int,
-    ip_a: int,
-    ip_b: int,
-) -> None:
-    """Add bidirectional topology edges between two nodes."""
-    await le_sender.send(
-        make_topology_edge_event(
-            node_a, node_b, sender_id, session_id, origin_idx_start, ip_suffix=ip_b
-        )
-    )
-    await le_sender.send(
-        make_topology_edge_event(
-            node_b, node_a, sender_id, session_id, origin_idx_start + 1, ip_suffix=ip_a
-        )
-    )
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_multi_node_sharding_distributes_layers() -> None:
-    """Place a model with min_nodes=2 on a cluster with 2 connected nodes.
-    Verify the resulting instance has shard assignments spanning both nodes.
-    """
-    master_nid = make_node_id("master-shard")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, _ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        worker_a = make_node_id("shard-worker-a")
-        worker_b = make_node_id("shard-worker-b")
-
-        # Register both worker nodes (each sender uses origin_idx=0)
-        for nid in [worker_a, worker_b]:
-            await _register_node(le_sender, nid, session_id)
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) < 2:
-                await anyio.sleep(0.01)
-
-        # Add bidirectional edges to form a 2-node cycle (A <-> B)
-        edge_sender = NodeId("edge_sender")
-        await _add_bidirectional_edge(
-            le_sender, worker_a, worker_b, session_id, edge_sender, 0, 1, 2
-        )
-
-        # Wait for edges to be processed
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_connections())) < 2:
-                await anyio.sleep(0.01)
-
-        # Place a large model requiring 2 nodes
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=LARGE_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=2,
-                ),
-            )
-        )
-
-        with anyio.fail_after(5):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        instance_id = next(iter(master.state.instances))
-        instance = master.state.instances[instance_id]
-        assert isinstance(instance, MlxRingInstance)
-
-        shard_assignments = instance.shard_assignments
-        runner_shards = shard_assignments.runner_to_shard
-
-        assert len(runner_shards) == 2
-
-        assigned_nodes = set(shard_assignments.node_to_runner.keys())
-        assert worker_a in assigned_nodes
-        assert worker_b in assigned_nodes
-
-        shards = list(runner_shards.values())
-        assert all(isinstance(s, PipelineShardMetadata) for s in shards)
-        pipeline_shards = [s for s in shards if isinstance(s, PipelineShardMetadata)]
-
-        assert all(s.world_size == 2 for s in pipeline_shards)
-        ranks = {s.device_rank for s in pipeline_shards}
-        assert ranks == {0, 1}
-
-        sorted_shards = sorted(pipeline_shards, key=lambda s: s.device_rank)
-        assert sorted_shards[0].start_layer == 0
-        assert sorted_shards[-1].end_layer == LARGE_MODEL_CARD.n_layers
-
-        total_layers = sum(s.end_layer - s.start_layer for s in sorted_shards)
-        assert total_layers == LARGE_MODEL_CARD.n_layers
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_single_node_gets_all_layers() -> None:
-    """Place a model with min_nodes=1 on a single node.  Verify the
-    instance has one runner assigned all layers (world_size=1).
-    """
-    master_nid = make_node_id("master-single")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, _ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        worker_nid = make_node_id("single-worker")
-        await _register_node(le_sender, worker_nid, session_id)
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) < 1:
-                await anyio.sleep(0.01)
-
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        instance_id = next(iter(master.state.instances))
-        instance = master.state.instances[instance_id]
-        assert isinstance(instance, MlxRingInstance)
-
-        shards = list(instance.shard_assignments.runner_to_shard.values())
-        assert len(shards) == 1
-
-        shard = shards[0]
-        assert isinstance(shard, PipelineShardMetadata)
-        assert shard.world_size == 1
-        assert shard.device_rank == 0
-        assert shard.start_layer == 0
-        assert shard.end_layer == TEST_MODEL_CARD.n_layers
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_three_node_sharding_distributes_evenly() -> None:
-    """Place a model across 3 connected nodes.  Verify all 3 get shard assignments."""
-    master_nid = make_node_id("master-3way")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, _ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        workers: list[NodeId] = []
-        for i in range(3):
-            nid = make_node_id(f"three-worker-{i}")
-            workers.append(nid)
-            await _register_node(le_sender, nid, session_id)
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) < 3:
-                await anyio.sleep(0.01)
-
-        # Add bidirectional edges to form a fully connected 3-node cycle:
-        # A <-> B, B <-> C, C <-> A
-        edge_sender = NodeId("edge_sender_3way")
-        idx = 0
-        ip_counter = 10
-        for i in range(3):
-            source = workers[i]
-            sink = workers[(i + 1) % 3]
-            # Forward edge
-            await le_sender.send(
-                make_topology_edge_event(
-                    source,
-                    sink,
-                    edge_sender,
-                    session_id,
-                    idx,
-                    ip_suffix=ip_counter,
-                )
-            )
-            idx += 1
-            ip_counter += 1
-            # Reverse edge
-            await le_sender.send(
-                make_topology_edge_event(
-                    sink,
-                    source,
-                    edge_sender,
-                    session_id,
-                    idx,
-                    ip_suffix=ip_counter,
-                )
-            )
-            idx += 1
-            ip_counter += 1
-
-        # Wait for all 6 edges (3 pairs x 2 directions)
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_connections())) < 6:
-                await anyio.sleep(0.01)
-
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=LARGE_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=3,
-                ),
-            )
-        )
-
-        with anyio.fail_after(5):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        instance = next(iter(master.state.instances.values()))
-        assert isinstance(instance, MlxRingInstance)
-
-        assignments = instance.shard_assignments
-        assert len(assignments.runner_to_shard) == 3
-        assert len(assignments.node_to_runner) == 3
-
-        for w in workers:
-            assert w in assignments.node_to_runner
-
-        shards = list(assignments.runner_to_shard.values())
-        ranks = {s.device_rank for s in shards if isinstance(s, PipelineShardMetadata)}
-        assert ranks == {0, 1, 2}
-
-        pipeline_shards = [s for s in shards if isinstance(s, PipelineShardMetadata)]
-        total_layers = sum(s.end_layer - s.start_layer for s in pipeline_shards)
-        assert total_layers == LARGE_MODEL_CARD.n_layers
-
-        await master.shutdown()
@@ -1,272 +0,0 @@
-"""E2E Chaos Test: Failure recovery.
-
-Scenarios:
-1. Master crash and re-election -- master shuts down, a new election round
-   produces a new master, workers re-converge.
-2. Worker crash during task execution -- runner death is detected, instance
-   is cleaned up, and cluster recovers.
-"""
-
-import anyio
-import pytest
-
-from exo.master.main import Master
-from exo.shared.types.commands import (
-    CommandId,
-    ForwarderCommand,
-    ForwarderDownloadCommand,
-    PlaceInstance,
-)
-from exo.shared.types.common import NodeId, SessionId
-from exo.shared.types.events import (
-    ForwarderEvent,
-    RunnerStatusUpdated,
-)
-from exo.shared.types.worker.instances import InstanceMeta
-from exo.shared.types.worker.runners import RunnerFailed
-from exo.shared.types.worker.shards import Sharding
-from exo.utils.channels import channel
-
-from .conftest import (
-    TEST_MODEL_CARD,
-    EventCollector,
-    MiniCluster,
-    make_gathered_info_event,
-    make_node_id,
-)
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_master_crash_and_reelection() -> None:
-    """Simulate master crash by shutting it down, then verify a new master
-    can be started with fresh state and begin accepting commands.
-
-    This tests the scenario where the elected master dies and a new election
-    must take place.  We simulate the election result directly (since
-    Election is tested separately) and verify the new master works.
-    """
-    cluster = MiniCluster(node_count=1)
-    old_instance_id: str = ""
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(cluster.master.run)
-
-        # Set up initial state
-        await cluster.inject_node_info(cluster.master_node_id)
-        await cluster.wait_for_topology_nodes(1)
-        await cluster.place_model()
-        await cluster.wait_for_instances(1)
-
-        # Verify initial state
-        assert len(cluster.master.state.instances) == 1
-        old_instance_id = next(iter(cluster.master.state.instances))
-
-        # --- Crash the master ---
-        await cluster.shutdown_master()
-
-    # --- Start a new master (simulating re-election) ---
-    new_master_nid = make_node_id("new-master")
-    new_session_id = SessionId(master_node_id=new_master_nid, election_clock=1)
-
-    ge_sender, ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    new_master = Master(
-        new_master_nid,
-        new_session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    _new_collector = EventCollector(ge_receiver.clone())
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(new_master.run)
-
-        # New master starts with clean state
-        assert len(new_master.state.instances) == 0
-        assert new_master.state.last_event_applied_idx == -1
-
-        # Re-register node with the new master
-        sender_id = NodeId(f"{new_master_nid}_sender_new")
-        await le_sender.send(
-            make_gathered_info_event(new_master_nid, sender_id, new_session_id, 0)
-        )
-
-        # Wait for topology to be rebuilt
-        with anyio.fail_after(3):
-            while len(list(new_master.state.topology.list_nodes())) == 0:
-                await anyio.sleep(0.01)
-
-        # Place a new model instance on the new master
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=new_master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(new_master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        # Verify new master is functional
-        assert len(new_master.state.instances) == 1
-        new_instance_id = next(iter(new_master.state.instances))
-        # New instance should be different from old one
-        assert new_instance_id != old_instance_id
-
-        await new_master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_runner_failure_triggers_instance_cleanup() -> None:
-    """Simulate a runner failure by injecting a RunnerStatusUpdated(RunnerFailed)
-    event.  Verify that the master's plan loop eventually detects the broken
-    instance (no connected node for the runner) and cleans it up.
-    """
-    master_nid = make_node_id("master-runner-fail")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    _collector = EventCollector(ge_receiver.clone())
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register a worker node
-        worker_nid = make_node_id("worker-failing")
-        sender_id = NodeId(f"{worker_nid}_sender")
-        await le_sender.send(
-            make_gathered_info_event(worker_nid, sender_id, session_id, 0)
-        )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) == 0:
-                await anyio.sleep(0.01)
-
-        # Place a model instance
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        instance_id = next(iter(master.state.instances))
-        instance = master.state.instances[instance_id]
-        runner_id = next(iter(instance.shard_assignments.runner_to_shard))
-
-        # Inject a RunnerFailed event from the worker
-        await le_sender.send(
-            ForwarderEvent(
-                origin_idx=1,
-                origin=sender_id,
-                session=session_id,
-                event=RunnerStatusUpdated(
-                    runner_id=runner_id,
-                    runner_status=RunnerFailed(
-                        error_message="Simulated OOM kill (exitcode=137)"
-                    ),
-                ),
-            )
-        )
-
-        # Wait for the runner failure to be processed
-        with anyio.fail_after(3):
-            while runner_id not in master.state.runners:
-                await anyio.sleep(0.01)
-
-        # The runner status should be RunnerFailed
-        assert isinstance(master.state.runners[runner_id], RunnerFailed)
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_election_recovers_after_multiple_node_joins() -> None:
-    """Verify that the election protocol correctly handles rapid node
-    join/leave events by running multiple election rounds.
-    """
-    from exo.routing.connection_message import ConnectionMessage, ConnectionMessageType
-    from exo.shared.election import Election, ElectionMessage, ElectionResult
-
-    em_out_tx, em_out_rx = channel[ElectionMessage]()
-    em_in_tx, em_in_rx = channel[ElectionMessage]()
-    er_tx, er_rx = channel[ElectionResult]()
-    cm_tx, cm_rx = channel[ConnectionMessage]()
-    co_tx, co_rx = channel[ForwarderCommand]()
-
-    election = Election(
-        node_id=NodeId("SURVIVOR"),
-        election_message_receiver=em_in_rx,
-        election_message_sender=em_out_tx,
-        election_result_sender=er_tx,
-        connection_message_receiver=cm_rx,
-        command_receiver=co_rx,
-        is_candidate=True,
-    )
-
-    async with anyio.create_task_group() as tg:
-        with anyio.fail_after(5):
-            tg.start_soon(election.run)
-
-            # Simulate rapid node joins via connection messages
-            for i in range(3):
-                await cm_tx.send(
-                    ConnectionMessage(
-                        node_id=NodeId(f"joiner-{i}"),
-                        connection_type=ConnectionMessageType.Connected,
-                        remote_ipv4=f"10.0.0.{i + 1}",
-                        remote_tcp_port=52415,
-                    )
-                )
-                # Each connection triggers a new election round
-                while True:
-                    got = await em_out_rx.receive()
-                    if got.proposed_session.master_node_id == NodeId("SURVIVOR"):
-                        break
-
-            # After all joins, an election result should eventually be produced
-            result = await er_rx.receive()
-            assert result.session_id.master_node_id == NodeId("SURVIVOR")
-
-            em_in_tx.close()
-            cm_tx.close()
-            co_tx.close()
@@ -1,227 +0,0 @@
-"""E2E Chaos Test: Networking resilience.
-
-Scenarios:
-1. Node disconnect mid-inference -- a worker stops receiving global events, then
-   reconnects and catches up via the event buffer / nack mechanism.
-2. Master detects stale node and times it out, then the node re-announces.
-"""
-
-import anyio
-import pytest
-
-from exo.master.main import Master
-from exo.shared.types.commands import (
-    ForwarderCommand,
-    ForwarderDownloadCommand,
-)
-from exo.shared.types.common import NodeId, SessionId
-from exo.shared.types.events import (
-    ForwarderEvent,
-    InstanceCreated,
-    NodeGatheredInfo,
-    TaskCreated,
-)
-from exo.utils.channels import channel
-
-from .conftest import (
-    EventCollector,
-    MiniCluster,
-    make_gathered_info_event,
-    make_node_id,
-)
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_node_disconnect_and_reconnect_event_replay() -> None:
-    """Simulate a node disconnecting by closing its global event receiver,
-    then reconnecting with a fresh receiver.
-
-    After reconnection, events that were broadcast while the node was
-    disconnected should be replayed to the new receiver via the shared
-    channel state.  The master's state should remain consistent.
-    """
-    cluster = MiniCluster(node_count=1)
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(cluster.master.run)
-
-        # Register the master node so topology is populated
-        await cluster.inject_node_info(cluster.master_node_id)
-        await cluster.wait_for_topology_nodes(1)
-
-        # Place a model instance
-        await cluster.place_model()
-        await cluster.wait_for_instances(1)
-
-        # Verify instance was created
-        assert len(cluster.master.state.instances) == 1
-
-        # --- Simulate disconnection ---
-        # The worker's global event receiver is independent; we just verify
-        # that the master continues to accept commands while a worker is gone.
-        _first_instance_id = next(iter(cluster.master.state.instances))
-
-        # Send a chat command while "disconnected" worker can't process
-        _cmd_id = await cluster.send_chat("Hello during disconnect")
-
-        # Give master time to process the command
-        await cluster.event_collector.wait_for_event_count(3, timeout=3.0)
-
-        events = cluster.event_collector.indexed_events
-        # Should have: NodeGatheredInfo, InstanceCreated, TaskCreated
-        assert any(isinstance(e.event, NodeGatheredInfo) for e in events)
-        assert any(isinstance(e.event, InstanceCreated) for e in events)
-        assert any(isinstance(e.event, TaskCreated) for e in events)
-
-        # --- Simulate reconnection ---
-        # A reconnecting node gets a fresh receiver clone and catches up
-        reconnect_receiver = cluster.global_event_internal_receiver.clone()
-        _reconnect_collector = EventCollector(reconnect_receiver)
-
-        # The new receiver should see future events; existing events are in
-        # the master's event log (which would be replayed via RequestEventLog
-        # in production). Here we verify the channel infrastructure works.
-        await cluster.send_chat("Hello after reconnect")
-        await anyio.sleep(0.1)
-
-        # Master state should now have 2 tasks
-        assert len(cluster.master.state.tasks) == 2
-
-        # The master's state is consistent throughout
-        assert len(cluster.master.state.instances) == 1
-        assert cluster.master.state.last_event_applied_idx >= 3
-
-        await cluster.shutdown_master()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_master_detects_timed_out_node_and_cleans_state() -> None:
-    """Verify that the master's plan loop detects a node that hasn't sent
-    a heartbeat (NodeGatheredInfo) recently and emits NodeTimedOut, cleaning
-    up topology and related state.
-    """
-    master_nid = make_node_id("master-timeout")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, ge_receiver = channel[ForwarderEvent]()
-    _cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    _collector = EventCollector(ge_receiver.clone())
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register two nodes
-        stale_node = make_node_id("stale")
-        alive_node = make_node_id("alive")
-
-        for node_id, suffix in [(stale_node, "_s0"), (alive_node, "_a0")]:
-            sender_id = NodeId(f"{node_id}_sender{suffix}")
-            await le_sender.send(
-                make_gathered_info_event(node_id, sender_id, session_id, 0)
-            )
-
-        # Wait for both nodes in topology
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) < 2:
-                await anyio.sleep(0.01)
-
-        assert stale_node in master.state.last_seen
-        assert alive_node in master.state.last_seen
-
-        # Manually expire the stale node's last_seen time by patching the state
-        # (in production, the _plan loop checks every 10s with a 30s threshold)
-        from datetime import timedelta
-
-        old_time = master.state.last_seen[stale_node] - timedelta(seconds=60)
-        patched_last_seen = {**master.state.last_seen, stale_node: old_time}
-        master.state = master.state.model_copy(update={"last_seen": patched_last_seen})
-
-        # Trigger the plan loop manually to speed up the test
-        # The plan loop checks for stale nodes
-        # We wait for the NodeTimedOut event to be emitted
-        with anyio.fail_after(15):
-            while stale_node in master.state.last_seen:
-                await anyio.sleep(0.1)
-
-        # Stale node should be removed from topology
-        assert stale_node not in set(master.state.topology.list_nodes())
-
-        # Alive node should still be present
-        assert alive_node in set(master.state.topology.list_nodes())
-        assert alive_node in master.state.last_seen
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_event_ordering_preserved_under_concurrent_writers() -> None:
-    """Multiple sources writing local events concurrently.  Verify that the
-    master's MultiSourceBuffer correctly sequences events from each source
-    and the final state is consistent.
-    """
-    master_nid = make_node_id("master-ordering")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, ge_receiver = channel[ForwarderEvent]()
-    _cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    _collector = EventCollector(ge_receiver.clone())
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Inject events from 3 different "worker" sources concurrently
-        node_ids = [make_node_id(f"concurrent-{i}") for i in range(3)]
-
-        async def inject_events(node_id: NodeId, count: int) -> None:
-            for idx in range(count):
-                sender_id = NodeId(f"{node_id}_sender")
-                await le_sender.send(
-                    make_gathered_info_event(node_id, sender_id, session_id, idx)
-                )
-                await anyio.sleep(0.001)  # slight jitter
-
-        async with anyio.create_task_group() as inject_tg:
-            for nid in node_ids:
-                inject_tg.start_soon(inject_events, nid, 5)
-
-        # Wait for master to process all events (3 nodes * 5 events each = 15)
-        with anyio.fail_after(5):
-            while master.state.last_event_applied_idx < 14:
-                await anyio.sleep(0.01)
-
-        # All 3 nodes should be visible in topology
-        topo_nodes = set(master.state.topology.list_nodes())
-        for nid in node_ids:
-            assert nid in topo_nodes
-
-        # Event indices should be sequential with no gaps
-        assert master.state.last_event_applied_idx == 14
-
-        await master.shutdown()
@@ -1,267 +0,0 @@
-"""E2E Chaos Test: Node join/leave during operation.
-
-Scenarios:
-1. Add nodes dynamically -- register new nodes with the master while
-   a model is already placed, verify topology grows.
-2. Remove nodes -- simulate node timeout, verify instances on that node
-   are cleaned up and remaining nodes are unaffected.
-3. Rapid join/leave churn -- nodes join and leave quickly, verify state
-   converges to a consistent snapshot.
-"""
-
-from datetime import timedelta
-
-import anyio
-import pytest
-
-from exo.master.main import Master
-from exo.shared.types.commands import (
-    CommandId,
-    ForwarderCommand,
-    ForwarderDownloadCommand,
-    PlaceInstance,
-)
-from exo.shared.types.common import NodeId, SessionId
-from exo.shared.types.events import (
-    ForwarderEvent,
-)
-from exo.shared.types.worker.instances import InstanceMeta
-from exo.shared.types.worker.shards import Sharding
-from exo.utils.channels import channel
-
-from .conftest import (
-    TEST_MODEL_CARD,
-    make_gathered_info_event,
-    make_node_id,
-)
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_dynamic_node_registration_expands_topology() -> None:
-    """Start with one node, then add more dynamically.  Verify the topology
-    grows and all nodes are visible in state.
-    """
-    master_nid = make_node_id("master-join")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, _ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register initial node
-        initial_node = make_node_id("initial")
-        sender_id = NodeId(f"{initial_node}_sender")
-        await le_sender.send(
-            make_gathered_info_event(initial_node, sender_id, session_id, 0)
-        )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) < 1:
-                await anyio.sleep(0.01)
-
-        # Place a model instance
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        # Dynamically add 3 more nodes
-        new_nodes: list[NodeId] = []
-        for i in range(3):
-            new_nid = make_node_id(f"dynamic-{i}")
-            new_nodes.append(new_nid)
-            new_sender = NodeId(f"{new_nid}_sender")
-            await le_sender.send(
-                make_gathered_info_event(new_nid, new_sender, session_id, 0)
-            )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) < 4:
-                await anyio.sleep(0.01)
-
-        # All 4 nodes should be in topology
-        topo_nodes = set(master.state.topology.list_nodes())
-        assert initial_node in topo_nodes
-        for nid in new_nodes:
-            assert nid in topo_nodes
-
-        # Original instance should still exist
-        assert len(master.state.instances) >= 1
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_node_removal_cleans_up_instances() -> None:
-    """Place a model on a specific node, then time it out.  Verify the
-    instance assigned to that node is deleted by the master's plan loop.
-    """
-    master_nid = make_node_id("master-leave")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, _ge_receiver = channel[ForwarderEvent]()
-    cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register a worker node
-        worker_nid = make_node_id("worker-leaving")
-        sender_id = NodeId(f"{worker_nid}_sender")
-        await le_sender.send(
-            make_gathered_info_event(worker_nid, sender_id, session_id, 0)
-        )
-
-        with anyio.fail_after(3):
-            while len(list(master.state.topology.list_nodes())) < 1:
-                await anyio.sleep(0.01)
-
-        # Place instance on the worker node
-        await cmd_sender.send(
-            ForwarderCommand(
-                origin=master_nid,
-                command=PlaceInstance(
-                    command_id=CommandId(),
-                    model_card=TEST_MODEL_CARD,
-                    sharding=Sharding.Pipeline,
-                    instance_meta=InstanceMeta.MlxRing,
-                    min_nodes=1,
-                ),
-            )
-        )
-
-        with anyio.fail_after(3):
-            while len(master.state.instances) == 0:
-                await anyio.sleep(0.01)
-
-        assert len(master.state.instances) == 1
-
-        # Simulate node leaving by expiring its last_seen
-        old_time = master.state.last_seen[worker_nid] - timedelta(seconds=60)
-        patched_last_seen = {**master.state.last_seen, worker_nid: old_time}
-        master.state = master.state.model_copy(update={"last_seen": patched_last_seen})
-
-        # The plan loop should detect the stale node and delete the instance
-        # because the node assigned to the instance is no longer in the topology
-        with anyio.fail_after(15):
-            while worker_nid in master.state.last_seen:
-                await anyio.sleep(0.1)
-
-        # After timeout, the node should be removed from topology
-        assert worker_nid not in set(master.state.topology.list_nodes())
-
-        # The instance should eventually be deleted since the assigned node
-        # is no longer connected (the _plan loop kills broken instances)
-        with anyio.fail_after(15):
-            while len(master.state.instances) > 0:
-                await anyio.sleep(0.1)
-
-        assert len(master.state.instances) == 0
-
-        await master.shutdown()
-
-
-@pytest.mark.slow
-@pytest.mark.asyncio
-async def test_rapid_join_leave_churn_converges() -> None:
-    """Rapidly join and leave nodes.  After the churn settles, verify the
-    master's state reflects only the surviving nodes.
-    """
-    master_nid = make_node_id("master-churn")
-    session_id = SessionId(master_node_id=master_nid, election_clock=0)
-
-    ge_sender, _ge_receiver = channel[ForwarderEvent]()
-    _cmd_sender, cmd_receiver = channel[ForwarderCommand]()
-    le_sender, le_receiver = channel[ForwarderEvent]()
-    dl_sender, _dl_receiver = channel[ForwarderDownloadCommand]()
-
-    master = Master(
-        master_nid,
-        session_id,
-        global_event_sender=ge_sender,
-        local_event_receiver=le_receiver,
-        command_receiver=cmd_receiver,
-        download_command_sender=dl_sender,
-    )
-
-    async with anyio.create_task_group() as tg:
-        tg.start_soon(master.run)
-
-        # Register 5 nodes rapidly
-        all_nodes: list[NodeId] = []
-        for i in range(5):
-            nid = make_node_id(f"churn-{i}")
-            all_nodes.append(nid)
-            sender_id = NodeId(f"{nid}_sender")
-            await le_sender.send(
-                make_gathered_info_event(nid, sender_id, session_id, 0)
-            )
-
-        with anyio.fail_after(5):
-            while len(list(master.state.topology.list_nodes())) < 5:
-                await anyio.sleep(0.01)
-
-        assert len(list(master.state.topology.list_nodes())) == 5
-
-        # Expire the first 3 nodes (simulate leaving)
-        leaving_nodes = all_nodes[:3]
-        surviving_nodes = all_nodes[3:]
-
-        patched_last_seen = dict(master.state.last_seen)
-        for nid in leaving_nodes:
-            patched_last_seen[nid] = patched_last_seen[nid] - timedelta(seconds=60)
-        master.state = master.state.model_copy(update={"last_seen": patched_last_seen})
-
-        # Wait for master's plan loop to time out the expired nodes
-        with anyio.fail_after(15):
-            while any(nid in master.state.last_seen for nid in leaving_nodes):
-                await anyio.sleep(0.1)
-
-        # Verify only surviving nodes remain
-        topo_nodes = set(master.state.topology.list_nodes())
-        for nid in leaving_nodes:
-            assert nid not in topo_nodes
-        for nid in surviving_nodes:
-            assert nid in topo_nodes
-
-        assert len(list(master.state.topology.list_nodes())) == 2
-
-        await master.shutdown()
@@ -1,5 +1,7 @@
+import sys
+
+
 def print_startup_banner(port: int) -> None:
-    """Print a prominent startup banner with API endpoint information."""
    dashboard_url = f"http://localhost:{port}"
    banner = f"""
 ╔═══════════════════════════════════════════════════════════════════════╗
@@ -27,4 +29,4 @@ def print_startup_banner(port: int) -> None:

 """

-    print(banner)
+    print(banner, file=sys.stderr)
@@ -306,7 +306,7 @@ def mlx_generate(
    max_stop_len = max((len(s) for s in stop_sequences), default=0)

    mx_barrier(group)
-    logger.info("Ready to prefill")
+    logger.info("Starting prefill")

    # Prefill cache with all tokens except the last one
    prefill_tps, prefill_tokens, ssm_snapshots_list = prefill(
@@ -353,7 +353,13 @@ def load_tokenizer_for_model_id(
            return list(hf_tokenizer.model.encode(text, allowed_special="all"))  # pyright: ignore[reportUnknownMemberType,reportUnknownArgumentType]

        hf_tokenizer.encode = _patched_encode
-        return TokenizerWrapper(hf_tokenizer, eos_token_ids=eos_token_ids)
+        return TokenizerWrapper(
+            hf_tokenizer,
+            eos_token_ids=eos_token_ids,
+            tool_call_start="<|tool_calls_section_begin|>",
+            tool_call_end="<|tool_calls_section_end|>",
+            tool_parser=_parse_kimi_tool_calls,
+        )

    tokenizer = load_tokenizer(
        model_path,
@@ -585,3 +591,41 @@ def mx_barrier(group: Group | None):
            mx.array(1.0), group=group, stream=mx.default_stream(mx.Device(mx.cpu))
        )
    )
+
+
+def _parse_kimi_tool_calls(text: str):
+    import regex as re
+
+    # kimi has a fixed function naming scheme, with a json formatted arg
+    #   functions.multiply:0<|tool_call_argument_begin|>{"a": 2, "b": 3}
+    _func_name_regex = re.compile(
+        r"^\s*((?:functions\.)?(.+?):\d+)\s*<\|tool_call_argument_begin\|>", re.DOTALL
+    )
+    _func_arg_regex = re.compile(r"<\|tool_call_argument_begin\|>\s*(.*)\s*", re.DOTALL)
+    _tool_call_split_regex = re.compile(
+        r"<\|tool_call_begin\|>(.*?)<\|tool_call_end\|>", re.DOTALL
+    )
+
+    def _parse_single_tool(text: str) -> dict[str, Any]:
+        func_name_match = _func_name_regex.search(text)
+        if func_name_match is None:
+            raise ValueError("No tool call found.")
+        tool_call_id = func_name_match.group(1)  # e.g. "functions.get_weather:0"
+        func_name = func_name_match.group(2)  # e.g. "get_weather"
+
+        func_args_match = _func_arg_regex.search(text)
+        if func_args_match is None:
+            raise ValueError("No tool call arguments found.")
+        func_args = func_args_match.group(1)
+        try:
+            arg_dct = json.loads(func_args)  # pyright: ignore[reportAny]
+        except Exception:
+            arg_dct = None
+
+        return dict(id=tool_call_id, name=func_name, arguments=arg_dct)
+
+    tool_matches = _tool_call_split_regex.findall(text)
+    if tool_matches:
+        return [_parse_single_tool(match) for match in tool_matches]  # pyright: ignore[reportAny]
+    else:
+        return [_parse_single_tool(text)]
@@ -348,7 +348,7 @@ class Worker:
                    session=self.session_id,
                    event=event,
                )
-                logger.debug(f"Worker published event {idx}: {str(event)[:100]}")
+                logger.trace(f"Worker published event {idx}: {str(event)[:100]}")
                await self.local_event_sender.send(fe)
                self.out_for_delivery[event.event_id] = fe

@@ -1,8 +1,4 @@
-from __future__ import annotations
-
 import os
-import threading
-from multiprocessing.sharedctypes import Synchronized

 import loguru

@@ -14,15 +10,6 @@ from exo.utils.channels import ClosedResourceError, MpReceiver, MpSender

 logger: "loguru.Logger" = loguru.logger

-HEARTBEAT_INTERVAL_SECONDS = 0.5
-
-
-def _heartbeat_loop(heartbeat: Synchronized[int], stop: threading.Event) -> None:
-    """Daemon thread that periodically increments the heartbeat counter."""
-    while not stop.is_set():
-        heartbeat.value += 1
-        stop.wait(HEARTBEAT_INTERVAL_SECONDS)
-

 def entrypoint(
    bound_instance: BoundInstance,
@@ -30,7 +17,6 @@ def entrypoint(
    task_receiver: MpReceiver[Task],
    cancel_receiver: MpReceiver[TaskId],
    _logger: "loguru.Logger",
-    heartbeat: Synchronized[int] | None = None,
 ) -> None:
    fast_synch_override = os.environ.get("EXO_FAST_SYNCH")
    if fast_synch_override == "on" or (
@@ -49,17 +35,6 @@ def entrypoint(

    logger.info(f"Fast synch flag: {os.environ['MLX_METAL_FAST_SYNCH']}")

-    # Start heartbeat thread so the supervisor can detect if we freeze.
-    stop_heartbeat = threading.Event()
-    heartbeat_thread: threading.Thread | None = None
-    if heartbeat is not None:
-        heartbeat_thread = threading.Thread(
-            target=_heartbeat_loop,
-            args=(heartbeat, stop_heartbeat),
-            daemon=True,
-        )
-        heartbeat_thread.start()
-
    # Import main after setting global logger - this lets us just import logger from this module
    try:
        from exo.worker.runner.runner import main
@@ -78,9 +53,6 @@ def entrypoint(
            )
        )
    finally:
-        stop_heartbeat.set()
-        if heartbeat_thread is not None:
-            heartbeat_thread.join(timeout=1)
        try:
            event_sender.close()
            task_receiver.close()
@@ -1,12 +1,10 @@
 import base64
-import json
 import math
-import os
 import resource
 import time
 from collections.abc import Generator
 from functools import cache
-from typing import Any, Callable, Literal
+from typing import Literal

 import mlx.core as mx
 from mlx_lm.models.gpt_oss import Model as GptOssModel
@@ -17,7 +15,6 @@ from openai_harmony import (  # pyright: ignore[reportMissingTypeStubs]
    StreamableParser,
    load_harmony_encoding,
 )
-from pydantic import ValidationError

 from exo.shared.constants import EXO_MAX_CHUNK_SIZE, EXO_TRACING_ENABLED
 from exo.shared.models.model_cards import ModelId, ModelTask
@@ -94,6 +91,8 @@ from exo.worker.engines.mlx.utils_mlx import (
 )
 from exo.worker.runner.bootstrap import logger

+from .tool_parsers import ToolParser, make_mlx_parser
+

 def _is_primary_output_node(shard_metadata: ShardMetadata) -> bool:
    """Check if this node is the primary output node for image generation.
@@ -139,6 +138,7 @@ def main(
    inference_model: Model | None = None
    image_model: DistributedImageModel | None = None
    tokenizer = None
+    tool_parser: ToolParser | None = None
    group = None
    kv_prefix_cache: KVPrefixCache | None = None
    check_for_cancel_every: int | None = None
@@ -204,8 +204,17 @@ def main(
                            bound_instance, group, on_timeout=on_model_load_timeout
                        )
                        logger.info(
-                            f"model has_tool_calling={tokenizer.has_tool_calling}"
+                            f"model has_tool_calling={tokenizer.has_tool_calling} using tokens {tokenizer.tool_call_start}, {tokenizer.tool_call_end}"
                        )
+                        if tokenizer.has_tool_calling:
+                            assert tokenizer.tool_call_start
+                            assert tokenizer.tool_call_end
+                            assert tokenizer.tool_parser  # pyright: ignore[reportAny]
+                            tool_parser = make_mlx_parser(
+                                tokenizer.tool_call_start,
+                                tokenizer.tool_call_end,
+                                tokenizer.tool_parser,  # pyright: ignore[reportAny]
+                            )
                        kv_prefix_cache = KVPrefixCache(group)

                    elif (
@@ -311,31 +320,11 @@ def main(
                                mlx_generator, tokenizer
                            )

-                        # Kimi-K2 has tool call sections - we don't care about them
-                        if "kimi" in shard_metadata.model_card.model_id.lower():
-                            mlx_generator = filter_kimi_tokens(mlx_generator)
-                            patch_kimi_tokenizer(tokenizer)
-
-                        # GLM models need patched parser (upstream has bug with None regex match)
-                        elif "glm" in shard_metadata.model_card.model_id.lower():
-                            patch_glm_tokenizer(tokenizer)
-
                        # GPT-OSS specific parsing to match other model formats.
-                        elif isinstance(inference_model, GptOssModel):
+                        if isinstance(inference_model, GptOssModel):
                            mlx_generator = parse_gpt_oss(mlx_generator)
-
-                        if tokenizer.has_tool_calling and not isinstance(
-                            inference_model, GptOssModel
-                        ):
-                            assert tokenizer.tool_call_start
-                            assert tokenizer.tool_call_end
-                            assert tokenizer.tool_parser  # pyright: ignore[reportAny]
-                            mlx_generator = parse_tool_calls(
-                                mlx_generator,
-                                tokenizer.tool_call_start,
-                                tokenizer.tool_call_end,
-                                tokenizer.tool_parser,  # pyright: ignore[reportAny]
-                            )
+                        elif tool_parser:
+                            mlx_generator = parse_tool_calls(mlx_generator, tool_parser)

                        completion_tokens = 0
                        tokens_since_last_cancel_check = 0
@@ -588,21 +577,8 @@ def get_gpt_oss_encoding():
    return encoding


-def filter_kimi_tokens(
-    responses: Generator[GenerationResponse | ToolCallResponse],
-) -> Generator[GenerationResponse]:
-    for resp in responses:
-        assert isinstance(resp, GenerationResponse)
-        if (
-            resp.text == "<|tool_calls_section_begin|>"
-            or resp.text == "<|tool_calls_section_end|>"
-        ):
-            continue
-        yield resp
-
-
 def parse_gpt_oss(
-    responses: Generator[GenerationResponse | ToolCallResponse],
+    responses: Generator[GenerationResponse],
 ) -> Generator[GenerationResponse | ToolCallResponse]:
    encoding = get_gpt_oss_encoding()
    stream = StreamableParser(encoding, role=Role.ASSISTANT)
@@ -659,9 +635,9 @@ def parse_gpt_oss(


 def parse_thinking_models(
-    responses: Generator[GenerationResponse | ToolCallResponse],
+    responses: Generator[GenerationResponse],
    tokenizer: TokenizerWrapper,
-) -> Generator[GenerationResponse | ToolCallResponse]:
+) -> Generator[GenerationResponse]:
    """
    For models that inject thinking tags in the prompt (like GLM-4.7),
    prepend the thinking tag to the output stream so the frontend
@@ -782,225 +758,58 @@ def _process_image_response(


 def parse_tool_calls(
-    responses: Generator[GenerationResponse | ToolCallResponse],
-    tool_call_start: str,
-    tool_call_end: str,
-    tool_parser: Callable[[str], dict[str, Any] | list[dict[str, Any]]],
+    responses: Generator[GenerationResponse], tool_parser: ToolParser
 ) -> Generator[GenerationResponse | ToolCallResponse]:
    in_tool_call = False
    tool_call_text_parts: list[str] = []
    for response in responses:
-        assert isinstance(response, GenerationResponse)
-        # assumption: the tool call start is one token
-        if response.text == tool_call_start:
+        if response.text.startswith(tool_parser.start_parsing):
            in_tool_call = True
-            continue
-        # assumption: the tool call end is one token
-        if in_tool_call and response.text == tool_call_end:
-            try:
-                # tool_parser returns an arbitrarily nested python dictionary
-                # we actually don't want the python dictionary, we just want to
-                # parse the top level { function: ..., arguments: ... } structure
-                # as we're just gonna hand it back to the api anyway
-                parsed = tool_parser("".join(tool_call_text_parts).strip())
-                logger.info(f"parsed {tool_call_text_parts=} into {parsed=}")
-                if isinstance(parsed, list):
-                    tools = [_validate_single_tool(tool) for tool in parsed]
-                else:
-                    tools = [_validate_single_tool(parsed)]
-                yield ToolCallResponse(
-                    tool_calls=tools, usage=response.usage, stats=response.stats
-                )
-
-            except (
-                json.JSONDecodeError,
-                ValidationError,
-                ValueError,
-                AttributeError,
-            ) as e:
-                # ValueError: our parsers raise this for malformed tool calls
-                # AttributeError: upstream parsers (e.g. glm47) may raise this when regex doesn't match
-                logger.opt(exception=e).warning("tool call parsing failed")
-                # assumption: talking about tool calls, not making a tool call
-                response.text = (
-                    tool_call_start + "".join(tool_call_text_parts) + tool_call_end
-                )
-                yield response
-
-            in_tool_call = False
-            tool_call_text_parts = []
-            continue

        if in_tool_call:
            tool_call_text_parts.append(response.text)
+            if response.text.endswith(tool_parser.end_parsing):
+                # parse the actual tool calls from the tool call text
+                parsed = tool_parser.parse_tool_calls(
+                    "".join(tool_call_text_parts).strip()
+                )
+                logger.info(f"parsed {tool_call_text_parts=} into {parsed=}")
+                if parsed is not None:
+                    yield ToolCallResponse(
+                        tool_calls=parsed, usage=response.usage, stats=response.stats
+                    )
+                else:
+                    logger.warning(
+                        f"tool call parsing failed for text {''.join(tool_call_text_parts)}"
+                    )
+                    response.text = "".join(tool_call_text_parts)
+                    yield response
+
+                in_tool_call = False
+                tool_call_text_parts = []
+                continue
+
            if response.finish_reason is not None:
                logger.info(
-                    "toll call parsing interrupted, yield partial tool call as text"
+                    "tool call parsing interrupted, yield partial tool call as text"
                )
-                yield GenerationResponse(
-                    text=tool_call_start + "".join(tool_call_text_parts),
-                    token=0,
-                    finish_reason=response.finish_reason,
-                    usage=response.usage,
-                    stats=response.stats,
+                response = response.model_copy(
+                    update={
+                        "text": "".join(tool_call_text_parts),
+                        "token": 0,
+                    }
                )
+                yield response
+
            continue
+
        # fallthrough
        yield response


-def patch_kimi_tokenizer(tokenizer: TokenizerWrapper):
-    """
-    Version of to-be-upstreamed kimi-k2 tool parser
-    """
-    import ast
-    import json
-    from typing import Any
-
-    import regex as re
-
-    # kimi has a fixed function naming scheme, with a json formatted arg
-    #   functions.multiply:0 <|tool_call_argument_begin|> {"a": 2, "b": 3}
-    #   Also needs to handle tools like call_0<|tool_call_argument_begin|>{"filePath": "..."}
-    _func_name_regex = re.compile(
-        r"^\s*(.+)[:](\d+)\s*<\|tool_call_argument_begin\|>", re.DOTALL
-    )
-    _func_arg_regex = re.compile(r"<\|tool_call_argument_begin\|>\s*(.*)\s*", re.DOTALL)
-
-    # kimi has a tool_calls_section - we're leaving this up to the caller to handle
-    tool_call_start = "<|tool_call_begin|>"
-    tool_call_end = "<|tool_call_end|>"
-
-    def _deserialize(value: str) -> Any:  # pyright: ignore[reportAny]
-        try:
-            return json.loads(value)  # pyright: ignore[reportAny]
-        except Exception:
-            pass
-
-        try:
-            return ast.literal_eval(value)  # pyright: ignore[reportAny]
-        except Exception:
-            pass
-        return value
-
-    def parse_tool_call(text: str, tools: Any | None = None):
-        func_name_match = _func_name_regex.search(text)
-        if func_name_match is None:
-            raise ValueError(f"Could not parse function name from tool call: {text!r}")
-        original_func_name = func_name_match.group(1)
-        tool_id = func_name_match.group(2)
-        # strip off the `functions.` prefix, if it exists.
-        func_name = original_func_name[original_func_name.find(".") + 1 :]
-
-        func_args_match = _func_arg_regex.search(text)
-        if func_args_match is None:
-            raise ValueError(f"Could not parse function args from tool call: {text!r}")
-        func_args = func_args_match.group(1)
-        # the args should be valid json - no need to check against our tools to deserialize
-        arg_dct = _deserialize(func_args)  # pyright: ignore[reportAny]
-
-        return dict(
-            id=f"{original_func_name}:{tool_id}",
-            name=func_name,
-            arguments=arg_dct,  # pyright: ignore[reportAny]
-        )
-
-    tokenizer._tool_call_start = tool_call_start
-    tokenizer._tool_call_end = tool_call_end
-    tokenizer._tool_parser = parse_tool_call
-
-
-def patch_glm_tokenizer(tokenizer: TokenizerWrapper):
-    """
-    Fixed version of mlx_lm's glm47 tool parser that handles regex match failures.
-    """
-    import ast
-    import json
-    from typing import Any
-
-    import regex as re
-
-    _func_name_regex = re.compile(r"^(.*?)<arg_key>", re.DOTALL)
-    _func_arg_regex = re.compile(
-        r"<arg_key>(.*?)</arg_key>(?:\n|\s)*<arg_value>(.*?)(?:</arg_value>|(?=<arg_key>)|$)",
-        re.DOTALL,
-    )
-
-    tool_call_start = "<tool_call>"
-    tool_call_end = "</tool_call>"
-
-    def _is_string_type(
-        tool_name: str,
-        arg_name: str,
-        tools: list[Any] | None,
-    ) -> bool:
-        if tools is None:
-            return False
-        for tool in tools:  # pyright: ignore[reportAny]
-            func = tool["function"]  # pyright: ignore[reportAny]
-            if func["name"] == tool_name:
-                params = func["parameters"]  # pyright: ignore[reportAny]
-                if params is None:
-                    return False
-                props = params.get("properties", {})  # pyright: ignore[reportAny]
-                arg_props = props.get(arg_name, {})  # pyright: ignore[reportAny]
-                arg_type = arg_props.get("type", None)  # pyright: ignore[reportAny]
-                return arg_type == "string"  # pyright: ignore[reportAny]
-        return False
-
-    def _deserialize(value: str) -> Any:  # pyright: ignore[reportAny]
-        try:
-            return json.loads(value)  # pyright: ignore[reportAny]
-        except Exception:
-            pass
-        try:
-            return ast.literal_eval(value)  # pyright: ignore[reportAny]
-        except Exception:
-            pass
-        return value
-
-    def parse_tool_call(text: str, tools: list[Any] | None = None):
-        func_name_match = _func_name_regex.search(text)
-        if func_name_match is None:
-            raise ValueError(f"Could not parse function name from tool call: {text!r}")
-        func_name = func_name_match.group(1)
-
-        pairs = _func_arg_regex.findall(text)
-        arg_dct: dict[str, Any] = {}
-        for key, value in pairs:  # pyright: ignore[reportAny]
-            arg_key = key.strip()  # pyright: ignore[reportAny]
-            arg_val = value.strip()  # pyright: ignore[reportAny]
-            if not _is_string_type(func_name, arg_key, tools):  # pyright: ignore[reportAny]
-                arg_val = _deserialize(arg_val)  # pyright: ignore[reportAny]
-            arg_dct[arg_key] = arg_val
-        return dict(name=func_name, arguments=arg_dct)
-
-    tokenizer._tool_call_start = tool_call_start
-    tokenizer._tool_call_end = tool_call_end
-    tokenizer._tool_parser = parse_tool_call
-
-
-def _validate_single_tool(obj: dict[str, Any]) -> ToolCallItem:
-    if (
-        ((name := obj.get("name")) is not None)
-        and ((args := obj.get("arguments")) is not None)
-        and isinstance(name, str)
-    ):
-        raw_id: object = obj.get("id")
-        extra = {"id": str(raw_id)} if raw_id is not None else {}
-        return ToolCallItem(
-            **extra,
-            name=name,
-            arguments=json.dumps(args),
-        )
-    else:
-        raise ValidationError
-
-
 EXO_RUNNER_MUST_FAIL = "EXO RUNNER MUST FAIL"
 EXO_RUNNER_MUST_OOM = "EXO RUNNER MUST OOM"
 EXO_RUNNER_MUST_TIMEOUT = "EXO RUNNER MUST TIMEOUT"
-EXO_RUNNER_MUST_DIE = "EXO RUNNER MUST DIE"


 def _check_for_debug_prompts(task_params: TextGenerationTaskParams) -> None:
@@ -1016,9 +825,6 @@ def _check_for_debug_prompts(task_params: TextGenerationTaskParams) -> None:
    if not prompt:
        return

-    if EXO_RUNNER_MUST_DIE in prompt:
-        logger.info("Abrupt process death triggered (simulates OOM kill)")
-        os._exit(1)
    if EXO_RUNNER_MUST_FAIL in prompt:
        logger.info("raising exception")
        raise Exception("Artificial runner exception - for testing purposes only.")
@@ -1,17 +1,12 @@
-from __future__ import annotations
-
 import contextlib
-import multiprocessing
 import signal
 from dataclasses import dataclass, field
 from multiprocessing import Process
-from multiprocessing.sharedctypes import Synchronized
 from typing import Self

 import anyio
 from anyio import (
    BrokenResourceError,
-    CancelScope,
    ClosedResourceError,
    to_thread,
 )
@@ -31,7 +26,6 @@ from exo.shared.types.worker.runners import (
    RunnerIdle,
    RunnerLoading,
    RunnerRunning,
-    RunnerShutdown,
    RunnerShuttingDown,
    RunnerStatus,
    RunnerWarmingUp,
@@ -42,8 +36,6 @@ from exo.worker.runner.bootstrap import entrypoint

 PREFILL_TIMEOUT_SECONDS = 60
 DECODE_TIMEOUT_SECONDS = 5
-HEALTH_CHECK_INTERVAL_SECONDS = 1
-HEARTBEAT_STALE_CHECKS = 10


@dataclass(eq=False)
@@ -56,14 +48,10 @@ class RunnerSupervisor:
    _task_sender: MpSender[Task]
    _event_sender: Sender[Event]
    _cancel_sender: MpSender[TaskId]
-    _heartbeat: Synchronized[int]
    status: RunnerStatus = field(default_factory=RunnerIdle, init=False)
    pending: dict[TaskId, anyio.Event] = field(default_factory=dict, init=False)
    completed: set[TaskId] = field(default_factory=set, init=False)
    cancelled: set[TaskId] = field(default_factory=set, init=False)
-    _death_handled: bool = field(default=False, init=False)
-    _last_heartbeat_value: int = field(default=0, init=False)
-    _heartbeat_stale_count: int = field(default=0, init=False)

    @classmethod
    def create(
@@ -77,8 +65,6 @@ class RunnerSupervisor:
        task_sender, task_recv = mp_channel[Task]()
        cancel_sender, cancel_recv = mp_channel[TaskId]()

-        heartbeat: Synchronized[int] = multiprocessing.Value("Q", 0)
-
        runner_process = Process(
            target=entrypoint,
            args=(
@@ -87,7 +73,6 @@ class RunnerSupervisor:
                task_recv,
                cancel_recv,
                logger,
-                heartbeat,
            ),
            daemon=True,
        )
@@ -103,16 +88,13 @@ class RunnerSupervisor:
            _task_sender=task_sender,
            _cancel_sender=cancel_sender,
            _event_sender=event_sender,
-            _heartbeat=heartbeat,
        )

        return self

    async def run(self):
        self.runner_process.start()
-        async with anyio.create_task_group() as tg:
-            tg.start_soon(self._forward_events)
-            tg.start_soon(self._health_check, tg.cancel_scope)
+        await self._forward_events()

    def shutdown(self):
        logger.info("Runner supervisor shutting down")
@@ -195,99 +177,9 @@ class RunnerSupervisor:
                        self.completed.add(event.task_id)
                    await self._event_sender.send(event)
            except (ClosedResourceError, BrokenResourceError) as e:
-                if not self._death_handled:
-                    self._death_handled = True
-                    await self._check_runner(e)
-                    for tid in self.pending:
-                        self.pending[tid].set()
-
-    async def _health_check(self, cancel_scope: CancelScope) -> None:
-        """Periodically check if the runner process is alive and responsive.
-
-        Detects two failure modes:
-        1. Process death (e.g. OOM kill) without cleanly closing the event
-           channel, which would leave _forward_events blocked on queue.get().
-        2. Unresponsive process (e.g. frozen by OS memory pressure, deadlock)
-           detected via a stale heartbeat counter.
-        """
-        while True:
-            await anyio.sleep(HEALTH_CHECK_INTERVAL_SECONDS)
-
-            if not self.runner_process.is_alive():
-                self._handle_process_exit(cancel_scope)
-                return
-
-            # Check heartbeat counter — if it hasn't changed between
-            # consecutive checks, the subprocess may be frozen.
-            current = self._heartbeat.value
-            if current > 0:
-                if current == self._last_heartbeat_value:
-                    self._heartbeat_stale_count += 1
-                    if self._heartbeat_stale_count >= HEARTBEAT_STALE_CHECKS:
-                        logger.error(
-                            f"Health check: runner process unresponsive "
-                            f"(heartbeat stale for {self._heartbeat_stale_count} checks), killing"
-                        )
-                        self._handle_unresponsive(cancel_scope)
-                        return
-                else:
-                    self._heartbeat_stale_count = 0
-                self._last_heartbeat_value = current
-
-    def _handle_process_exit(self, cancel_scope: CancelScope) -> None:
-        """Handle runner process that has exited."""
-        if not self._death_handled:
-            self._death_handled = True
-            if isinstance(
-                self.status, (RunnerShutdown, RunnerShuttingDown, RunnerFailed)
-            ):
-                logger.info("Health check: runner process exited (expected)")
-            else:
-                rc = self.runner_process.exitcode
-                if isinstance(rc, int) and rc < 0:
-                    sig = -rc
-                    try:
-                        cause = f"signal={sig} ({signal.strsignal(sig)})"
-                    except Exception:
-                        cause = f"signal={sig}"
-                else:
-                    cause = f"exitcode={rc}"
-
-                logger.error(
-                    f"Health check: runner process died unexpectedly ({cause})"
-                )
-                self._event_sender.send_nowait(
-                    RunnerStatusUpdated(
-                        runner_id=self.bound_instance.bound_runner_id,
-                        runner_status=RunnerFailed(
-                            error_message=f"Terminated ({cause})"
-                        ),
-                    )
-                )
-                self.shutdown()
-
-            for tid in self.pending:
-                self.pending[tid].set()
-
-        cancel_scope.cancel()
-
-    def _handle_unresponsive(self, cancel_scope: CancelScope) -> None:
-        """Handle runner process that is alive but unresponsive."""
-        if not self._death_handled:
-            self._death_handled = True
-            self._event_sender.send_nowait(
-                RunnerStatusUpdated(
-                    runner_id=self.bound_instance.bound_runner_id,
-                    runner_status=RunnerFailed(
-                        error_message="Runner process unresponsive (heartbeat timeout)"
-                    ),
-                )
-            )
-            for tid in self.pending:
-                self.pending[tid].set()
-            self.shutdown()
-
-        cancel_scope.cancel()
+                await self._check_runner(e)
+                for tid in self.pending:
+                    self.pending[tid].set()

    def __del__(self) -> None:
        if self.runner_process.is_alive():
@@ -0,0 +1,72 @@
+import json
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from exo.shared.types.api import ToolCallItem
+
+
+@dataclass
+class ToolParser:
+    start_parsing: str
+    end_parsing: str
+    parse_tool_calls: Callable[[str], list[ToolCallItem] | None]
+
+
+def make_mlx_parser(
+    tool_call_start: str,
+    tool_call_end: str,
+    tool_parser: Callable[[str], dict[str, Any] | list[dict[str, Any]]],
+) -> ToolParser:
+    def parse_tool_calls(text: str) -> list[ToolCallItem] | None:
+        try:
+            text = text.removeprefix(tool_call_start)
+            text = text.removesuffix(tool_call_end)
+            parsed = tool_parser(text)
+            if isinstance(parsed, list):
+                return [ToolCallItem.model_validate(_flatten(p)) for p in parsed]
+            else:
+                return [ToolCallItem.model_validate(_flatten(parsed))]
+
+        except Exception:
+            return None
+
+    return ToolParser(
+        start_parsing=tool_call_start,
+        end_parsing=tool_call_end,
+        parse_tool_calls=parse_tool_calls,
+    )
+
+
+# TODO / example code:
+def _parse_json_calls(text: str) -> list[ToolCallItem] | None:
+    try:
+        text = text.removeprefix("<tool_call>")
+        text = text.removesuffix("</tool_call>")
+        top_level = {
+            k: json.dumps(v) if isinstance(v, (dict, list)) else v
+            for k, v in json.loads(text).items()  # pyright: ignore[reportAny]
+        }
+        return [ToolCallItem.model_validate(top_level)]
+    except Exception:
+        return None
+
+
+def _flatten(p: dict[str, Any]) -> dict[str, str]:
+    return {
+        k: json.dumps(v) if isinstance(v, (dict, list)) else str(v)  # pyright: ignore[reportAny]
+        for k, v in p.items()  # pyright: ignore[reportAny]
+    }
+
+
+json_tool_parser = ToolParser(
+    start_parsing="<tool_call>",
+    end_parsing="</tool_call>",
+    parse_tool_calls=_parse_json_calls,
+)
+
+
+def infer_tool_parser(chat_template: str) -> ToolParser | None:
+    """Attempt to auto-infer a tool parser from the chat template."""
+    if "<tool_call>" in chat_template and "tool_call.name" in chat_template:
+        return json_tool_parser
+    return None
@@ -5,12 +5,13 @@ from typing import Any

 from exo.shared.types.worker.runner_response import GenerationResponse, ToolCallResponse
 from exo.worker.runner.runner import parse_tool_calls
+from exo.worker.runner.tool_parsers import make_mlx_parser


 def _make_responses(
    texts: list[str],
    finish_on_last: bool = True,
-) -> Generator[GenerationResponse | ToolCallResponse]:
+) -> Generator[GenerationResponse]:
    """Create a sequence of GenerationResponses from text strings."""
    for i, text in enumerate(texts):
        is_last = i == len(texts) - 1
@@ -22,10 +23,13 @@ def _make_responses(
        )


-def _dummy_parser(text: str) -> dict[str, Any]:
+def _dummier_parser(text: str) -> dict[str, Any]:
    return {"name": "test_fn", "arguments": {"arg": text}}


+_dummy_parser = make_mlx_parser("<tool_call>", "</tool_call>", _dummier_parser)
+
+
 class TestParseToolCalls:
    """Tests for parse_tool_calls generator."""

@@ -35,8 +39,6 @@ class TestParseToolCalls:
        results = list(
            parse_tool_calls(
                _make_responses(texts, finish_on_last=False),
-                "<tool_call>",
-                "</tool_call>",
                _dummy_parser,
            )
        )
@@ -50,8 +52,6 @@ class TestParseToolCalls:
        results = list(
            parse_tool_calls(
                _make_responses(texts),
-                "<tool_call>",
-                "</tool_call>",
                _dummy_parser,
            )
        )
@@ -76,9 +76,7 @@ class TestParseToolCalls:
        results = list(
            parse_tool_calls(
                _make_responses(texts, finish_on_last=False),
-                "<tool_call>",
-                "</tool_call>",
-                _failing_parser,
+                make_mlx_parser("<tool_call>", "</tool_call>", _failing_parser),
            )
        )

@@ -1,204 +1 @@
-from __future__ import annotations
-
-import multiprocessing
-import os
-import signal as signal_module
-from collections.abc import Callable
-from multiprocessing.sharedctypes import Synchronized
-from typing import Any
-
-import anyio
-
-from exo.shared.types.events import Event, RunnerStatusUpdated
-from exo.shared.types.tasks import Task, TaskId
-from exo.shared.types.worker.runners import RunnerFailed, RunnerIdle, RunnerShutdown
-from exo.utils.channels import Receiver, Sender, channel, mp_channel
-from exo.worker.runner.runner_supervisor import (
-    HEALTH_CHECK_INTERVAL_SECONDS,
-    HEARTBEAT_STALE_CHECKS,
-    RunnerSupervisor,
-)
-
-from ...constants import (
-    INSTANCE_1_ID,
-    MODEL_A_ID,
-    NODE_A,
-    RUNNER_1_ID,
-)
-from ..conftest import get_bound_mlx_ring_instance
-
-
-def _die_immediately() -> None:
-    """Subprocess target that exits with a non-zero code."""
-    os._exit(1)
-
-
-def _die_with_signal() -> None:
-    """Subprocess target that kills itself with SIGKILL (simulates OOM)."""
-    os.kill(os.getpid(), signal_module.SIGKILL)
-
-
-def _exit_cleanly() -> None:
-    """Subprocess target that exits with code 0."""
-    os._exit(0)
-
-
-def _hang_forever() -> None:
-    """Subprocess target that hangs without updating heartbeat (simulates freeze)."""
-    import time
-
-    # Write one heartbeat so the supervisor starts tracking, then stop.
-    time.sleep(100000)
-
-
-def _build_supervisor(
-    event_sender: Sender[Event],
-    target: Callable[..., Any],
-) -> RunnerSupervisor:
-    """Build a RunnerSupervisor with a custom subprocess target.
-
-    Uses a clone of event_sender (matching real Worker behavior) so that
-    closing the supervisor's copy doesn't close the test's receiver.
-    """
-    bound_instance = get_bound_mlx_ring_instance(
-        instance_id=INSTANCE_1_ID,
-        model_id=MODEL_A_ID,
-        runner_id=RUNNER_1_ID,
-        node_id=NODE_A,
-    )
-
-    _ev_send, ev_recv = mp_channel[Event]()
-    task_sender, _task_recv = mp_channel[Task]()
-    cancel_sender, _cancel_recv = mp_channel[TaskId]()
-    runner_process = multiprocessing.Process(target=target, daemon=True)
-    heartbeat: Synchronized[int] = multiprocessing.Value("Q", 0)
-
-    return RunnerSupervisor(
-        bound_instance=bound_instance,
-        shard_metadata=bound_instance.bound_shard,
-        runner_process=runner_process,
-        initialize_timeout=10,
-        _ev_recv=ev_recv,
-        _task_sender=task_sender,
-        _cancel_sender=cancel_sender,
-        _event_sender=event_sender.clone(),
-        _heartbeat=heartbeat,
-    )
-
-
-def _collect_failed_events(
-    event_receiver: Receiver[Event],
-) -> list[RunnerFailed]:
-    """Drain the receiver and return all RunnerFailed statuses."""
-    out: list[RunnerFailed] = []
-    while True:
-        try:
-            event = event_receiver.receive_nowait()
-        except Exception:
-            break
-        if isinstance(event, RunnerStatusUpdated) and isinstance(
-            event.runner_status, RunnerFailed
-        ):
-            out.append(event.runner_status)
-    return out
-
-
-async def test_health_check_detects_dead_process():
-    """When the runner process dies with a non-zero exit code, the health check
-    should emit a RunnerFailed event and run() should return."""
-    event_sender, event_receiver = channel[Event]()
-    supervisor = _build_supervisor(event_sender, _die_immediately)
-
-    with anyio.fail_after(HEALTH_CHECK_INTERVAL_SECONDS + 5):
-        await supervisor.run()
-
-    failures = _collect_failed_events(event_receiver)
-    assert len(failures) == 1
-    assert failures[0].error_message is not None
-    assert "exitcode=1" in failures[0].error_message
-
-
-async def test_health_check_detects_signal_death():
-    """When the runner process is killed by a signal (e.g. OOM -> SIGKILL),
-    the health check should report the signal in the failure message."""
-    event_sender, event_receiver = channel[Event]()
-    supervisor = _build_supervisor(event_sender, _die_with_signal)
-
-    with anyio.fail_after(HEALTH_CHECK_INTERVAL_SECONDS + 5):
-        await supervisor.run()
-
-    failures = _collect_failed_events(event_receiver)
-    assert len(failures) == 1
-    assert failures[0].error_message is not None
-    assert "signal=9" in failures[0].error_message
-
-
-async def test_health_check_releases_pending_tasks():
-    """When the runner dies, any pending start_task() waiters should be unblocked."""
-    event_sender, _event_receiver = channel[Event]()
-    supervisor = _build_supervisor(event_sender, _die_immediately)
-
-    # Register a pending waiter as if start_task() was waiting for acknowledgement
-    task_event = anyio.Event()
-    tid = TaskId("pending-task")
-    supervisor.pending[tid] = task_event
-
-    with anyio.fail_after(HEALTH_CHECK_INTERVAL_SECONDS + 5):
-        await supervisor.run()
-
-    assert task_event.is_set()
-
-
-async def test_clean_exit_no_failure_when_shutdown_status():
-    """When the runner was in RunnerShutdown status and exits with code 0,
-    no RunnerFailed event should be emitted."""
-    event_sender, event_receiver = channel[Event]()
-    supervisor = _build_supervisor(event_sender, _exit_cleanly)
-
-    # Simulate that the runner had already reported shutdown via events
-    supervisor.status = RunnerShutdown()
-
-    with anyio.fail_after(HEALTH_CHECK_INTERVAL_SECONDS + 5):
-        await supervisor.run()
-
-    failures = _collect_failed_events(event_receiver)
-    assert len(failures) == 0
-
-
-async def test_unexpected_exit_code_zero_emits_failure():
-    """When the runner exits with code 0 but was NOT in a shutdown state,
-    this is unexpected and should still emit RunnerFailed."""
-    event_sender, event_receiver = channel[Event]()
-    supervisor = _build_supervisor(event_sender, _exit_cleanly)
-
-    assert isinstance(supervisor.status, RunnerIdle)
-
-    with anyio.fail_after(HEALTH_CHECK_INTERVAL_SECONDS + 5):
-        await supervisor.run()
-
-    failures = _collect_failed_events(event_receiver)
-    assert len(failures) == 1
-    assert failures[0].error_message is not None
-    assert "exitcode=0" in failures[0].error_message
-
-
-async def test_heartbeat_timeout_detects_unresponsive_process():
-    """When the runner process is alive but its heartbeat goes stale,
-    the health check should kill it and emit RunnerFailed."""
-    event_sender, event_receiver = channel[Event]()
-    supervisor = _build_supervisor(event_sender, _hang_forever)
-
-    # Pre-seed the heartbeat counter with a non-zero value and set the
-    # supervisor's last-seen value to match so it appears stale immediately.
-    # Set stale count to HEARTBEAT_STALE_CHECKS - 1 so a single check triggers.
-    supervisor._heartbeat.value = 42  # pyright: ignore[reportPrivateUsage]
-    supervisor._last_heartbeat_value = 42  # pyright: ignore[reportPrivateUsage]
-    supervisor._heartbeat_stale_count = HEARTBEAT_STALE_CHECKS - 1  # pyright: ignore[reportPrivateUsage]
-
-    with anyio.fail_after(HEALTH_CHECK_INTERVAL_SECONDS + 5):
-        await supervisor.run()
-
-    failures = _collect_failed_events(event_receiver)
-    assert len(failures) == 1
-    assert failures[0].error_message is not None
-    assert "unresponsive" in failures[0].error_message.lower()
+# TODO:
@@ -28,12 +28,12 @@ trap 'cleanup' EXIT INT TERM

 for host; do
  ssh -T -o BatchMode=yes -o ServerAliveInterval=30 "$host@$host" \
-    "EXO_LIBP2P_NAMESPACE=$commit /nix/var/nix/profiles/default/bin/nix build github:exo-explore/exo/$commit" &
+    "/nix/var/nix/profiles/default/bin/nix build github:exo-explore/exo/$commit" &
 done
 wait
 for host; do
  ssh -T -o BatchMode=yes -o ServerAliveInterval=30 "$host@$host" \
-    "EXO_LIBP2P_NAMESPACE=$commit /nix/var/nix/profiles/default/bin/nix run github:exo-explore/exo/$commit" &>/dev/null &
+    "/nix/var/nix/profiles/default/bin/nix run github:exo-explore/exo/$commit -- --namespace $commit" &>/dev/null &
 done

 for host; do
@@ -35,7 +35,7 @@ i=0
 for host; do
  colour=${colours[i++ % 4]}
  ssh -T -o BatchMode=yes -o ServerAliveInterval=30 "$host@$host" \
-    "EXO_LIBP2P_NAMESPACE=$commit /nix/var/nix/profiles/default/bin/nix run github:exo-explore/exo/$commit" |&
+    "/nix/var/nix/profiles/default/bin/nix run github:exo-explore/exo/$commit -- --namespace $commit" |&
    awk -v p="${colour}[${host}]${reset}" '{ print p $0; fflush() }' &
 done

@@ -193,20 +193,14 @@ sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8
 wheels = [
    { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446, upload-time = "2025-09-08T23:23:03.472Z" },
    { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101, upload-time = "2025-09-08T23:23:04.792Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948, upload-time = "2025-09-08T23:23:06.127Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422, upload-time = "2025-09-08T23:23:07.753Z" },
    { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499, upload-time = "2025-09-08T23:23:09.648Z" },
    { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928, upload-time = "2025-09-08T23:23:10.928Z" },
    { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302, upload-time = "2025-09-08T23:23:12.42Z" },
    { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049, upload-time = "2025-09-08T23:23:20.853Z" },
-    { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793, upload-time = "2025-09-08T23:23:22.08Z" },
-    { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300, upload-time = "2025-09-08T23:23:23.314Z" },
    { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244, upload-time = "2025-09-08T23:23:24.541Z" },
    { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828, upload-time = "2025-09-08T23:23:26.143Z" },
    { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926, upload-time = "2025-09-08T23:23:27.873Z" },
    { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593, upload-time = "2025-09-08T23:23:31.91Z" },
-    { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354, upload-time = "2025-09-08T23:23:33.214Z" },
-    { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480, upload-time = "2025-09-08T23:23:34.495Z" },
    { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584, upload-time = "2025-09-08T23:23:36.096Z" },
    { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443, upload-time = "2025-09-08T23:23:37.328Z" },
    { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437, upload-time = "2025-09-08T23:23:38.945Z" },
@@ -312,10 +306,8 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/5c/49/498c86566a1d80e978b42f0d702795f69887005548c041636df6ae1ca64c/cryptography-46.0.3-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:01ca9ff2885f3acc98c29f1860552e37f6d7c7d013d7334ff2a9de43a449315d", size = 4450807, upload-time = "2025-10-15T23:16:56.414Z" },
    { url = "https://files.pythonhosted.org/packages/4b/0a/863a3604112174c8624a2ac3c038662d9e59970c7f926acdcfaed8d61142/cryptography-46.0.3-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6eae65d4c3d33da080cff9c4ab1f711b15c1d9760809dad6ea763f3812d254cb", size = 4299615, upload-time = "2025-10-15T23:16:58.442Z" },
    { url = "https://files.pythonhosted.org/packages/64/02/b73a533f6b64a69f3cd3872acb6ebc12aef924d8d103133bb3ea750dc703/cryptography-46.0.3-cp311-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5bf0ed4490068a2e72ac03d786693adeb909981cc596425d09032d372bcc849", size = 4016800, upload-time = "2025-10-15T23:17:00.378Z" },
-    { url = "https://files.pythonhosted.org/packages/25/d5/16e41afbfa450cde85a3b7ec599bebefaef16b5c6ba4ec49a3532336ed72/cryptography-46.0.3-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:5ecfccd2329e37e9b7112a888e76d9feca2347f12f37918facbb893d7bb88ee8", size = 4984707, upload-time = "2025-10-15T23:17:01.98Z" },
    { url = "https://files.pythonhosted.org/packages/c9/56/e7e69b427c3878352c2fb9b450bd0e19ed552753491d39d7d0a2f5226d41/cryptography-46.0.3-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a2c0cd47381a3229c403062f764160d57d4d175e022c1df84e168c6251a22eec", size = 4482541, upload-time = "2025-10-15T23:17:04.078Z" },
    { url = "https://files.pythonhosted.org/packages/78/f6/50736d40d97e8483172f1bb6e698895b92a223dba513b0ca6f06b2365339/cryptography-46.0.3-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:549e234ff32571b1f4076ac269fcce7a808d3bf98b76c8dd560e42dbc66d7d91", size = 4299464, upload-time = "2025-10-15T23:17:05.483Z" },
-    { url = "https://files.pythonhosted.org/packages/00/de/d8e26b1a855f19d9994a19c702fa2e93b0456beccbcfe437eda00e0701f2/cryptography-46.0.3-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:c0a7bb1a68a5d3471880e264621346c48665b3bf1c3759d682fc0864c540bd9e", size = 4950838, upload-time = "2025-10-15T23:17:07.425Z" },
    { url = "https://files.pythonhosted.org/packages/8f/29/798fc4ec461a1c9e9f735f2fc58741b0daae30688f41b2497dcbc9ed1355/cryptography-46.0.3-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:10b01676fc208c3e6feeb25a8b83d81767e8059e1fe86e1dc62d10a3018fa926", size = 4481596, upload-time = "2025-10-15T23:17:09.343Z" },
    { url = "https://files.pythonhosted.org/packages/15/8d/03cd48b20a573adfff7652b76271078e3045b9f49387920e7f1f631d125e/cryptography-46.0.3-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:0abf1ffd6e57c67e92af68330d05760b7b7efb243aab8377e583284dbab72c71", size = 4426782, upload-time = "2025-10-15T23:17:11.22Z" },
    { url = "https://files.pythonhosted.org/packages/fa/b1/ebacbfe53317d55cf33165bda24c86523497a6881f339f9aae5c2e13e57b/cryptography-46.0.3-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a04bee9ab6a4da801eb9b51f1b708a1b5b5c9eb48c03f74198464c66f0d344ac", size = 4698381, upload-time = "2025-10-15T23:17:12.829Z" },
@@ -323,10 +315,8 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/c5/fd/bc1daf8230eaa075184cbbf5f8cd00ba9db4fd32d63fb83da4671b72ed8a/cryptography-46.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:39b6755623145ad5eff1dab323f4eae2a32a77a7abef2c5089a04a3d04366715", size = 4435078, upload-time = "2025-10-15T23:17:23.042Z" },
    { url = "https://files.pythonhosted.org/packages/82/98/d3bd5407ce4c60017f8ff9e63ffee4200ab3e23fe05b765cab805a7db008/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:db391fa7c66df6762ee3f00c95a89e6d428f4d60e7abc8328f4fe155b5ac6e54", size = 4293460, upload-time = "2025-10-15T23:17:24.885Z" },
    { url = "https://files.pythonhosted.org/packages/26/e9/e23e7900983c2b8af7a08098db406cf989d7f09caea7897e347598d4cd5b/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:78a97cf6a8839a48c49271cdcbd5cf37ca2c1d6b7fdd86cc864f302b5e9bf459", size = 3995237, upload-time = "2025-10-15T23:17:26.449Z" },
-    { url = "https://files.pythonhosted.org/packages/91/15/af68c509d4a138cfe299d0d7ddb14afba15233223ebd933b4bbdbc7155d3/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:dfb781ff7eaa91a6f7fd41776ec37c5853c795d3b358d4896fdbb5df168af422", size = 4967344, upload-time = "2025-10-15T23:17:28.06Z" },
    { url = "https://files.pythonhosted.org/packages/ca/e3/8643d077c53868b681af077edf6b3cb58288b5423610f21c62aadcbe99f4/cryptography-46.0.3-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:6f61efb26e76c45c4a227835ddeae96d83624fb0d29eb5df5b96e14ed1a0afb7", size = 4466564, upload-time = "2025-10-15T23:17:29.665Z" },
    { url = "https://files.pythonhosted.org/packages/0e/43/c1e8726fa59c236ff477ff2b5dc071e54b21e5a1e51aa2cee1676f1c986f/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:23b1a8f26e43f47ceb6d6a43115f33a5a37d57df4ea0ca295b780ae8546e8044", size = 4292415, upload-time = "2025-10-15T23:17:31.686Z" },
-    { url = "https://files.pythonhosted.org/packages/42/f9/2f8fefdb1aee8a8e3256a0568cffc4e6d517b256a2fe97a029b3f1b9fe7e/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:b419ae593c86b87014b9be7396b385491ad7f320bde96826d0dd174459e54665", size = 4931457, upload-time = "2025-10-15T23:17:33.478Z" },
    { url = "https://files.pythonhosted.org/packages/79/30/9b54127a9a778ccd6d27c3da7563e9f2d341826075ceab89ae3b41bf5be2/cryptography-46.0.3-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:50fc3343ac490c6b08c0cf0d704e881d0d660be923fd3076db3e932007e726e3", size = 4466074, upload-time = "2025-10-15T23:17:35.158Z" },
    { url = "https://files.pythonhosted.org/packages/ac/68/b4f4a10928e26c941b1b6a179143af9f4d27d88fe84a6a3c53592d2e76bf/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:22d7e97932f511d6b0b04f2bfd818d73dcd5928db509460aaf48384778eb6d20", size = 4420569, upload-time = "2025-10-15T23:17:37.188Z" },
    { url = "https://files.pythonhosted.org/packages/a3/49/3746dab4c0d1979888f125226357d3262a6dd40e114ac29e3d2abdf1ec55/cryptography-46.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:d55f3dffadd674514ad19451161118fd010988540cee43d8bc20675e775925de", size = 4681941, upload-time = "2025-10-15T23:17:39.236Z" },
@@ -334,10 +324,8 @@ wheels = [
    { url = "https://files.pythonhosted.org/packages/26/42/fa8389d4478368743e24e61eea78846a0006caffaf72ea24a15159215a14/cryptography-46.0.3-cp38-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:15ab9b093e8f09daab0f2159bb7e47532596075139dd74365da52ecc9cb46c5d", size = 4440029, upload-time = "2025-10-15T23:17:49.837Z" },
    { url = "https://files.pythonhosted.org/packages/5f/eb/f483db0ec5ac040824f269e93dd2bd8a21ecd1027e77ad7bdf6914f2fd80/cryptography-46.0.3-cp38-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:46acf53b40ea38f9c6c229599a4a13f0d46a6c3fa9ef19fc1a124d62e338dfa0", size = 4297222, upload-time = "2025-10-15T23:17:51.357Z" },
    { url = "https://files.pythonhosted.org/packages/fd/cf/da9502c4e1912cb1da3807ea3618a6829bee8207456fbbeebc361ec38ba3/cryptography-46.0.3-cp38-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10ca84c4668d066a9878890047f03546f3ae0a6b8b39b697457b7757aaf18dbc", size = 4012280, upload-time = "2025-10-15T23:17:52.964Z" },
-    { url = "https://files.pythonhosted.org/packages/6b/8f/9adb86b93330e0df8b3dcf03eae67c33ba89958fc2e03862ef1ac2b42465/cryptography-46.0.3-cp38-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:36e627112085bb3b81b19fed209c05ce2a52ee8b15d161b7c643a7d5a88491f3", size = 4978958, upload-time = "2025-10-15T23:17:54.965Z" },
    { url = "https://files.pythonhosted.org/packages/d1/a0/5fa77988289c34bdb9f913f5606ecc9ada1adb5ae870bd0d1054a7021cc4/cryptography-46.0.3-cp38-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:1000713389b75c449a6e979ffc7dcc8ac90b437048766cef052d4d30b8220971", size = 4473714, upload-time = "2025-10-15T23:17:56.754Z" },
    { url = "https://files.pythonhosted.org/packages/14/e5/fc82d72a58d41c393697aa18c9abe5ae1214ff6f2a5c18ac470f92777895/cryptography-46.0.3-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:b02cf04496f6576afffef5ddd04a0cb7d49cf6be16a9059d793a30b035f6b6ac", size = 4296970, upload-time = "2025-10-15T23:17:58.588Z" },
-    { url = "https://files.pythonhosted.org/packages/78/06/5663ed35438d0b09056973994f1aec467492b33bd31da36e468b01ec1097/cryptography-46.0.3-cp38-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:71e842ec9bc7abf543b47cf86b9a743baa95f4677d22baa4c7d5c69e49e9bc04", size = 4940236, upload-time = "2025-10-15T23:18:00.897Z" },
    { url = "https://files.pythonhosted.org/packages/fc/59/873633f3f2dcd8a053b8dd1d38f783043b5fce589c0f6988bf55ef57e43e/cryptography-46.0.3-cp38-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:402b58fc32614f00980b66d6e56a5b4118e6cb362ae8f3fda141ba4689bd4506", size = 4472642, upload-time = "2025-10-15T23:18:02.749Z" },
    { url = "https://files.pythonhosted.org/packages/3d/39/8e71f3930e40f6877737d6f69248cf74d4e34b886a3967d32f919cc50d3b/cryptography-46.0.3-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ef639cb3372f69ec44915fafcd6698b6cc78fbe0c2ea41be867f6ed612811963", size = 4423126, upload-time = "2025-10-15T23:18:04.85Z" },
    { url = "https://files.pythonhosted.org/packages/cd/c7/f65027c2810e14c3e7268353b1681932b87e5a48e65505d8cc17c99e36ae/cryptography-46.0.3-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b51b8ca4f1c6453d8829e1eb7299499ca7f313900dd4d89a24b8b87c0a780d4", size = 4686573, upload-time = "2025-10-15T23:18:06.908Z" },
Author	SHA1	Message	Date
Evan	45a9f32fc7	woahg	2026-02-17 11:44:23 +00:00
Evan	c01b6fff21	eprint banner our banner was being printed to stdout but should be printed to stderr as its essentially a log message	2026-02-17 11:43:06 +00:00
Jake Hillion	8392e78afe	bench: add spec for automatic canary benchmarks (#1483 ) Adds all the models that can fit onto a single M3 Ultra for single machine benchmarks. Fixes the macOS version, GPU spec, and chip type for maximum reproducibility. Specifies the minimum memory accordingly for each type of model, using the smallest machine available (the smallest M3 Ultra is 96GiB). Test plan: - Running this with some code that makes machines of this spec available and stores the results. It works. This will become part of a larger testing/stability strategy once we've collected more of the data.	2026-02-17 10:52:05 +00:00
Evan	86735ece78	begins begins	2026-02-16 19:26:19 +00:00
				`@@ -1 +0,0 @@`
				`collect_ignore = ["tests/start_distributed_test.py"]`