feat: make snapshot tests run on x86 Ubuntu CI without GPU

MLX already supports x86 CPU via mlx[cpu] and the Dockerfile has the
GCC workaround for CPU JIT. The only barriers were the 'slow' markers
causing tests to be skipped in CI.

Changes:
- Remove 'slow' marker from all snapshot tests so they run by default
- Make snapshots architecture-aware (snapshots/{arch}/{name}.json) since
  floating-point results differ between x86_64 and arm64
- Store architecture in snapshot metadata
- Increase CI timeout from 30 to 45 minutes for model download + CPU inference
- Update docstrings to remove Apple Silicon requirement

Co-Authored-By: Claude Opus 4.6 <[email protected]>
This commit is contained in:
Alex Cheema
2026-02-16 10:00:04 -08:00
co-authored by Claude Opus 4.6
parent 3fb663ec25
commit effafc1d48
7 changed files with 16 additions and 21 deletions
+1 -1
View File
@@ -10,7 +10,7 @@ on:
jobs:
e2e:
runs-on: ubuntu-latest
timeout-minutes: 30
timeout-minutes: 45
steps:
- name: Free up disk space
run: |
+12 -7
View File
@@ -3,14 +3,19 @@
Provides deterministic regression testing by comparing inference output
against saved snapshots. On first run, snapshots are created automatically.
Set UPDATE_SNAPSHOTS=1 to regenerate snapshots when output intentionally changes.
Snapshots are stored per-architecture (e.g. snapshots/x86_64/, snapshots/arm64/)
since floating-point results differ between CPU architectures.
"""
import difflib
import json
import os
import platform
from pathlib import Path
SNAPSHOTS_DIR = Path(__file__).parent / "snapshots"
ARCH = platform.machine()
SNAPSHOTS_DIR = Path(__file__).parent / "snapshots" / ARCH
def assert_snapshot(
@@ -21,7 +26,7 @@ def assert_snapshot(
"""Compare content against a saved snapshot, or create one if missing.
Args:
name: Snapshot identifier (used as filename: snapshots/{name}.json).
name: Snapshot identifier (used as filename: snapshots/{arch}/{name}.json).
content: The actual inference output to compare.
metadata: Additional context stored alongside content (model, seed, etc.).
Not used for comparison -- purely documentary.
@@ -49,16 +54,16 @@ def assert_snapshot(
)
)
raise AssertionError(
f"Snapshot mismatch for '{name}'!\n\n"
f"Snapshot mismatch for '{name}' on {ARCH}!\n\n"
f"{diff}\n\n"
f"Expected: {expected!r}\n"
f"Actual: {content!r}\n\n"
f"To update: UPDATE_SNAPSHOTS=1 python3 e2e/run_all.py --slow"
f"To update: UPDATE_SNAPSHOTS=1 python3 e2e/run_all.py"
)
print(f" Output matches snapshot ({snapshot_file.name})")
print(f" Output matches snapshot ({ARCH}/{snapshot_file.name})")
else:
SNAPSHOTS_DIR.mkdir(parents=True, exist_ok=True)
snapshot_data = {**metadata, "content": content}
snapshot_data = {**metadata, "arch": ARCH, "content": content}
snapshot_file.write_text(json.dumps(snapshot_data, indent=2) + "\n")
action = "Updated" if update else "Created"
print(f" {action} snapshot: {snapshot_file}")
print(f" {action} snapshot: {ARCH}/{snapshot_file.name}")
+1 -3
View File
@@ -1,12 +1,10 @@
"""Test: Deterministic inference output (snapshot test).
slow
Sends a chat completion request with a fixed seed,
then verifies the output matches a known-good snapshot. This ensures
inference produces consistent results across runs.
Requires a machine that can run MLX inference at reasonable speed (Apple Silicon).
Run with: python3 e2e/run_all.py --slow or E2E_SLOW=1 python3 e2e/run_all.py
Uses MLX CPU backend in Docker on x86 Linux.
"""
import asyncio
-2
View File
@@ -1,8 +1,6 @@
"""Test: Code generation snapshot.
slow
Verifies deterministic output for a code generation prompt.
Run with: python3 e2e/run_all.py --slow or E2E_SLOW=1 python3 e2e/run_all.py
"""
import asyncio
+2 -4
View File
@@ -1,9 +1,7 @@
"""Test: Edge case snapshots.
slow
Verifies deterministic output for edge-case prompts: empty-ish input,
very short input, and special characters.
Run with: python3 e2e/run_all.py --slow or E2E_SLOW=1 python3 e2e/run_all.py
Verifies deterministic output for edge-case prompts: single word input,
special characters, and unicode.
"""
import asyncio
-2
View File
@@ -1,8 +1,6 @@
"""Test: Longer output snapshot.
slow
Verifies deterministic output with a higher max_tokens (128).
Run with: python3 e2e/run_all.py --slow or E2E_SLOW=1 python3 e2e/run_all.py
"""
import asyncio
-2
View File
@@ -1,8 +1,6 @@
"""Test: Reasoning/math snapshot.
slow
Verifies deterministic output for a simple reasoning prompt.
Run with: python3 e2e/run_all.py --slow or E2E_SLOW=1 python3 e2e/run_all.py
"""
import asyncio