diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..19890730 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,15 @@ +.venv/ +.direnv/ +target/ +.git/ +.idea/ +.pytest_cache/ +.ruff_cache/ +dashboard/node_modules/ +dashboard/.svelte-kit/ +dashboard/build/ +dist/ +*.pdb +**/__pycache__ +**/.DS_Store +.mlx_typings/ diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..5d7553fb --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,20 @@ +name: e2e-tests + +on: + push: + pull_request: + branches: + - staging + - main + +jobs: + e2e: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + lfs: false + + - name: Run E2E tests + run: python3 e2e/run_all.py diff --git a/e2e/Dockerfile b/e2e/Dockerfile new file mode 100644 index 00000000..40ddacf2 --- /dev/null +++ b/e2e/Dockerfile @@ -0,0 +1,51 @@ +# Stage 1: Build the dashboard +FROM node:22-slim AS dashboard +WORKDIR /app/dashboard +COPY dashboard/package.json dashboard/package-lock.json ./ +RUN npm ci +COPY dashboard/ . +RUN npm run build + +# Stage 2: Build and run exo +FROM python:3.13-slim + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + pkg-config \ + libssl-dev \ + curl \ + protobuf-compiler \ + && rm -rf /var/lib/apt/lists/* + +# Install Rust nightly +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install uv +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +WORKDIR /app + +# Copy dependency files first for better layer caching +COPY pyproject.toml Cargo.toml uv.lock README.md ./ +COPY rust/ ./rust/ +COPY bench/pyproject.toml ./bench/pyproject.toml + +# Copy source and resources +COPY src/ ./src/ +COPY resources/ ./resources/ + +# Copy built dashboard from stage 1 +COPY --from=dashboard /app/dashboard/build ./dashboard/build/ + +# Install Python deps and build Rust bindings +RUN uv sync + +# Wrap g++ with -fpermissive to fix MLX CPU JIT compilation with GCC 14 +# (GCC 14 treats _Float128/_Float32/_Float64 as built-in types, conflicting with MLX-generated code) +RUN mv /usr/bin/g++ /usr/bin/g++.real && \ + printf '#!/bin/sh\nexec /usr/bin/g++.real -fpermissive "$@"\n' > /usr/bin/g++ && \ + chmod +x /usr/bin/g++ + +CMD [".venv/bin/exo", "-v"] diff --git a/e2e/conftest.py b/e2e/conftest.py new file mode 100644 index 00000000..d86296ea --- /dev/null +++ b/e2e/conftest.py @@ -0,0 +1,101 @@ +"""Shared E2E test infrastructure for exo cluster tests.""" + +import asyncio +import os +import sys +from pathlib import Path +from urllib.request import urlopen +from urllib.error import URLError + +E2E_DIR = Path(__file__).parent.resolve() +TIMEOUT = int(os.environ.get("E2E_TIMEOUT", "120")) + + +class Cluster: + """Async wrapper around a docker compose exo cluster.""" + + def __init__(self, name: str, overrides: list[str] | None = None): + self.name = name + self.project = f"e2e-{name}" + compose_files = [str(E2E_DIR / "docker-compose.yml")] + for path in overrides or []: + compose_files.append(str(E2E_DIR / path)) + self._compose_base = [ + "docker", "compose", + "-p", self.project, + *[arg for f in compose_files for arg in ("-f", f)], + ] + + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + await self.stop() + + async def _run(self, *args: str, check: bool = True) -> str: + proc = await asyncio.create_subprocess_exec( + *self._compose_base, *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + stdout, _ = await proc.communicate() + output = stdout.decode() + if check and proc.returncode != 0: + print(output, file=sys.stderr) + raise RuntimeError(f"docker compose {' '.join(args)} failed (rc={proc.returncode})") + return output + + async def build(self): + print(" Building images...") + await self._run("build", "--quiet") + + async def start(self): + print(" Starting cluster...") + await self._run("up", "-d") + + async def stop(self): + print(" Cleaning up...") + await self._run("down", "--timeout", "5", check=False) + + async def logs(self) -> str: + return await self._run("logs", check=False) + + async def wait_for(self, description: str, check_fn, timeout: int = TIMEOUT): + """Poll check_fn every 2s until it returns True or timeout expires.""" + print(f" Waiting for {description}...") + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + if await check_fn(): + print(f" {description}") + return + await asyncio.sleep(2) + output = await self.logs() + print(f"--- cluster logs ---\n{output}\n---", file=sys.stderr) + raise TimeoutError(f"Timed out waiting for {description}") + + async def assert_healthy(self): + """Verify the cluster formed correctly: nodes started, discovered each other, elected a master, API responds.""" + + async def both_nodes_started(): + log = await self.logs() + return log.count("Starting node") >= 2 + + async def nodes_discovered(): + log = await self.logs() + return log.count("ConnectionMessageType.Connected") >= 2 + + async def master_elected(): + log = await self.logs() + return "demoting self" in log + + async def api_responding(): + try: + with urlopen("http://localhost:52415/v1/models", timeout=3) as resp: + return resp.status == 200 + except (URLError, OSError): + return False + + await self.wait_for("Both nodes started", both_nodes_started) + await self.wait_for("Nodes discovered each other", nodes_discovered) + await self.wait_for("Master election resolved", master_elected) + await self.wait_for("API responding", api_responding) diff --git a/e2e/docker-compose.yml b/e2e/docker-compose.yml new file mode 100644 index 00000000..e08d0620 --- /dev/null +++ b/e2e/docker-compose.yml @@ -0,0 +1,18 @@ +services: + exo-node-1: + build: + context: .. + dockerfile: e2e/Dockerfile + environment: + - EXO_LIBP2P_NAMESPACE=docker-e2e + command: [".venv/bin/exo", "-v"] + ports: + - "52415:52415" + + exo-node-2: + build: + context: .. + dockerfile: e2e/Dockerfile + environment: + - EXO_LIBP2P_NAMESPACE=docker-e2e + command: [".venv/bin/exo", "-v"] diff --git a/e2e/run_all.py b/e2e/run_all.py new file mode 100644 index 00000000..cd290623 --- /dev/null +++ b/e2e/run_all.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Discovers and runs all E2E tests in e2e/test_*.py.""" + +import subprocess +import sys +from pathlib import Path + +E2E_DIR = Path(__file__).parent.resolve() + + +def main(): + test_files = sorted(E2E_DIR.glob("test_*.py")) + if not test_files: + print("No test files found") + sys.exit(1) + + passed = 0 + failed = 0 + failures = [] + + for test_file in test_files: + name = test_file.stem + print(f"=== {name} ===") + result = subprocess.run([sys.executable, str(test_file)]) + if result.returncode == 0: + passed += 1 + else: + failed += 1 + failures.append(name) + print() + + total = passed + failed + print("================================") + print(f"{passed}/{total} tests passed") + + if failed: + print(f"Failed: {' '.join(failures)}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/e2e/test_cluster_formation.py b/e2e/test_cluster_formation.py new file mode 100644 index 00000000..444830d5 --- /dev/null +++ b/e2e/test_cluster_formation.py @@ -0,0 +1,21 @@ +"""Test: Basic cluster formation. + +Verifies two nodes discover each other, elect a master, and the API responds. +""" + +import asyncio +import sys +sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent)) +from conftest import Cluster + + +async def main(): + async with Cluster("cluster_formation") as cluster: + await cluster.build() + await cluster.start() + await cluster.assert_healthy() + print("PASSED: cluster_formation") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/e2e/test_no_internet.py b/e2e/test_no_internet.py new file mode 100644 index 00000000..5ecaaf7f --- /dev/null +++ b/e2e/test_no_internet.py @@ -0,0 +1,25 @@ +"""Test: Cluster works without internet access. + +Verifies exo functions correctly when containers can talk to each other +but cannot reach the internet (Docker internal network). +""" + +import asyncio +import sys +sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent)) +from conftest import Cluster + + +async def main(): + async with Cluster( + "no_internet", + overrides=["tests/no_internet/docker-compose.override.yml"], + ) as cluster: + await cluster.build() + await cluster.start() + await cluster.assert_healthy() + print("PASSED: no_internet") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/e2e/tests/no_internet/docker-compose.override.yml b/e2e/tests/no_internet/docker-compose.override.yml new file mode 100644 index 00000000..26a36ca2 --- /dev/null +++ b/e2e/tests/no_internet/docker-compose.override.yml @@ -0,0 +1,8 @@ +# Block DNS resolution to simulate no internet access. +# mDNS discovery (multicast 224.0.0.251) still works since it doesn't use DNS. +# Docker's "internal: true" can't be used here because it also blocks multicast. +services: + exo-node-1: + dns: 0.0.0.0 + exo-node-2: + dns: 0.0.0.0