electron-rare/exo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

we got it to BUILD

Browse Source
This commit is contained in:
Evan
2026-03-20 15:38:16 +00:00
parent 581a1fcd79
commit 532a8f0b07
42 changed files with 456 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
flake.nix
+4 -6
View File
@@ -76,15 +76,15 @@
let
# Use pinned nixpkgs for swift-format (swift is broken on x86_64-linux in newer nixpkgs)
pkgsSwift = import inputs.nixpkgs-swift { inherit system; };
pkgsCuda = import ./nix/cuda-pkgs.nix { nixpkgs = inputs.nixpkgs; inherit system; };
in
{
# Allow unfree for metal-toolchain (needed for Darwin Metal packages)
_module.args.pkgs = import inputs.nixpkgs {
inherit system;
config.allowUnfreePredicate = pkg: (pkg.pname or "") == "metal-toolchain";
overlays = [
config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ "metal-toolchain" "cuda-merged" "cuda_cuobjdump" "cuda_gdb" "cuda_nvcc" "cuda_nvdisasm" "cuda_nvprune" "cuda_cccl" "cuda_cudart" "cuda_cupti" "cuda_cuxxfilt" "cuda_nvml_dev" "cuda_nvrtc" "cuda_nvtx" "cuda_profiler_api" "cuda_sanitizer_api" "libcublas" "libcufft" "libcurand" "libcusolver" "libnvjitlink" "libcusparse" "libnpp" "cudnn" "libcusparse_lt" "libcufile" "libnvshmem" "libnvvm" "cuda_crt" ];
cudaSupport = true;
cudaCapabilities = [ "12.1" ];
overlays = lib.optionals (system == "aarch64-darwin") [
(import ./nix/apple-sdk-overlay.nix)
];
};
@@ -131,10 +131,8 @@
default = self'.packages.exo;
}
);
# CUDA development shell with torch + vLLM (aarch64-linux only)
devShells =
{
default = with pkgs; pkgs.mkShell {
inputsFrom = [ self'.checks.cargo-build ];
pyproject.toml
+10 -6
View File
@@ -45,21 +45,22 @@ dev = [
"pytest-env",
"ruff>=0.11.13",
]
build = [
"nanobind"
]
[project.optional-dependencies]
mlx = ["mlx_engine"]
cuda = ["vllm_engine; sys_platform == 'linux' and platform_machine == 'aarch64'"]
cuda = [
"vllm_engine; sys_platform == 'linux' and platform_machine == 'aarch64'",
]
###
# workspace configuration
###
[tool.uv.workspace]
members = [
"rust/exo_pyo3_bindings",
"bench",
"python/exo_core",
]
members = ["rust/exo_pyo3_bindings", "bench", "python/exo_core"]
[tool.uv.sources]
exo_pyo3_bindings = { workspace = true }
@@ -130,6 +131,9 @@ prerelease = "allow"
environments = ["sys_platform == 'darwin'", "sys_platform == 'linux'"]
conflicts = [[{ package = "exo", extra = "cuda" }, { package = "exo-bench" }]]
[tool.uv.extra-build-dependencies]
xgrammar = [ "nanobind" ]
###
# ruff configuration
python/exo_core/pyproject.toml
+2 -6
View File
@@ -3,13 +3,9 @@ name = "exo-core"
version = "0.1.0"
description = "Add your description here"
readme = "README.md"
authors = [
{ name = "Evan", email = "[email protected]" }
]
authors = [{ name = "Evan", email = "[email protected]" }]
requires-python = ">=3.13"
dependencies = [
"pydantic",
]
dependencies = ["pydantic"]
[build-system]
requires = ["uv_build>=0.9.24,<0.10.0"]
python/exo_core/src/exo_core/engine.py
+9 -4
View File
@@ -2,12 +2,19 @@ from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from typing import Self
class TaskId(str): ...
class Cancelled: ...
class Finished: ...
CANCEL_ALL_TASKS = TaskId("CANCEL_TALL_TASKS")
class Engine[TaskType, ResponseType](ABC):
_cancelled_tasks: set[TaskId]
@@ -32,13 +39,12 @@ class Engine[TaskType, ResponseType](ABC):
@abstractmethod
def step(
self,
) -> Iterable[
tuple[TaskId, ResponseType | Cancelled | Finished]
]: ...
) -> Iterable[tuple[TaskId, ResponseType | Cancelled | Finished]]: ...
@abstractmethod
def close(self) -> None: ...
class EngineBuilder[SetupType, TaskType, ResponseType](ABC):
@classmethod
@abstractmethod
@@ -62,4 +68,3 @@ class EngineBuilder[SetupType, TaskType, ResponseType](ABC):
@abstractmethod
def close(self) -> None: ...
python/exo_core/src/exo_core/model_cards.py
+3 -3
View File
@@ -5,6 +5,9 @@ import aiofiles
import aiofiles.os as aios
import tomlkit
from anyio import Path, open_file
from exo.shared.types.common import ModelId
from exo.shared.types.memory import Memory
from exo.utils.pydantic_ext import CamelCaseModel
from huggingface_hub import model_info
from loguru import logger
from pydantic import (
@@ -23,9 +26,6 @@ from exo.shared.constants import (
EXO_ENABLE_IMAGE_MODELS,
RESOURCES_DIR,
)
from exo.shared.types.common import ModelId
from exo.shared.types.memory import Memory
from exo.utils.pydantic_ext import CamelCaseModel
# kinda ugly...
# TODO: load search path from config.toml
python/exo_core/src/exo_core/types/instances.py
+2 -2
View File
@@ -1,10 +1,10 @@
from enum import Enum
from exo.shared.types.common import Host, Id, NodeId
from exo.shared.types.worker.runners import RunnerId, ShardAssignments, ShardMetadata
from pydantic import model_validator
from exo_core.model_cards import ModelTask
from exo.shared.types.common import Host, Id, NodeId
from exo.shared.types.worker.runners import RunnerId, ShardAssignments, ShardMetadata
from exo_core.pydantic_ext import CamelCaseModel, TaggedModel
python/exo_core/src/exo_core/types/runner_response.py
+2 -1
View File
@@ -1,6 +1,8 @@
from collections.abc import Generator
from typing import Any, Literal
from exo.utils.pydantic_ext import TaggedModel
from exo.api.types import (
FinishReason,
GenerationStats,
@@ -9,7 +11,6 @@ from exo.api.types import (
TopLogprobItem,
Usage,
)
from exo.utils.pydantic_ext import TaggedModel
class BaseRunnerResponse(TaggedModel):
python/exo_core/src/exo_core/types/runners.py
+1 -1
View File
@@ -3,9 +3,9 @@ from collections.abc import Mapping
from pydantic import model_validator
from exo_core.model_cards import ModelId
from exo_core.pydantic import CamelCaseModel, TaggedModel
from exo_core.types.common import Id, NodeId
from exo_core.types.shards import ShardMetadata
from exo_core.pydantic import CamelCaseModel, TaggedModel
class RunnerId(Id):
python/exo_core/src/exo_core/types/tasks.py
+2 -2
View File
@@ -6,12 +6,12 @@ from exo.api.types import (
ImageEditsTaskParams,
ImageGenerationTaskParams,
)
from exo_core.pydantic import TaggedModel
from exo_core.types.common import CommandId, Id
from exo_core.types.text_generation import TextGenerationTaskParams
from exo_core.types.instances import BoundInstance, InstanceId
from exo_core.types.runners import RunnerId
from exo_core.types.shards import ShardMetadata
from exo_core.pydantic import TaggedModel
from exo_core.types.text_generation import TextGenerationTaskParams
class TaskId(Id):
python/nanobind_cmake.patch
+25
View File
@@ -0,0 +1,25 @@
diff --git a/cpp/nanobind/CMakeLists.txt b/cpp/nanobind/CMakeLists.txt
index ebfd3da..6ef06e3 100644
--- a/cpp/nanobind/CMakeLists.txt
+++ b/cpp/nanobind/CMakeLists.txt
@@ -18,8 +18,18 @@ target_sources(python_methods PRIVATE python_methods.cc)
target_link_libraries(python_methods PUBLIC xgrammar)
# Any code that uses nanobind directly lives here
-nanobind_add_module(xgrammar_bindings LTO nanobind.cc)
-target_link_libraries(xgrammar_bindings PRIVATE python_methods)
+nanobind_build_library(nanobind)
+add_library(xgrammar_bindings MODULE nanobind.cc)
+target_link_libraries(xgrammar_bindings PRIVATE
+ python_methods
+ nanobind
+)
+nanobind_opt_size(xgrammar_bindings)
+nanobind_lto(xgrammar_bindings)
+nanobind_set_visibility(xgrammar_bindings)
+nanobind_extension(xgrammar_bindings)
+nanobind_compile_options(xgrammar_bindings)
+nanobind_link_options(xgrammar_bindings)
if(DEFINED SKBUILD_PROJECT_NAME)
# Building wheel through scikit-build-core
python/parts.nix
+224 -31
View File
@@ -38,38 +38,231 @@
python = pkgs.python313;
# Overlay to provide build systems and custom packages
buildSystemsOverlay = final: prev: {
# mlx-lm is a git dependency that needs setuptools
mlx-lm = prev.mlx-lm.overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
final.setuptools
buildSystemsOverlay = final: prev:
let
addSetupTools = (old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
final.setuptools
];
});
torchLibs = [
final.nvidia-cuda-runtime
final.nvidia-cuda-nvrtc
final.nvidia-cuda-cupti
final.nvidia-nvjitlink
final.nvidia-cudnn-cu13
final.nvidia-cusparse
final.nvidia-cusparselt-cu13
final.nvidia-cufile
final.nvidia-nvshmem-cu13
final.nvidia-nccl-cu13
final.nvidia-cublas
final.nvidia-cufft
final.nvidia-curand
final.nvidia-cusolver
];
});
# rouge-score and sacrebleu don't declare setuptools as a build dependency
rouge-score = prev.rouge-score.overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
final.setuptools
cutlass = pkgs.fetchFromGitHub {
name = "cutlass-source";
owner = "NVIDIA";
repo = "cutlass";
tag = "v4.2.1";
hash = "sha256-iP560D5Vwuj6wX1otJhwbvqe/X4mYVeKTpK533Wr5gY=";
};
triton-kernels = pkgs.fetchFromGitHub {
owner = "triton-lang";
repo = "triton";
tag = "v3.5.0";
hash = "sha256-F6T0n37Lbs+B7UHNYzoIQHjNNv3TcMtoXjNrT8ZUlxY=";
};
cutlass-flashmla = pkgs.fetchFromGitHub {
owner = "NVIDIA";
repo = "cutlass";
rev = "147f5673d0c1c3dcf66f78d677fd647e4a020219";
hash = "sha256-dHQto08IwTDOIuFUp9jwm1MWkFi8v2YJ/UESrLuG71g=";
};
flashmla = pkgs.stdenv.mkDerivation {
pname = "flashmla";
version = "1.0.0";
src = pkgs.fetchFromGitHub {
name = "FlashMLA-source";
owner = "vllm-project";
repo = "FlashMLA";
rev = "c2afa9cb93e674d5a9120a170a6da57b89267208";
hash = "sha256-pKlwxV6G9iHag/jbu3bAyvYvnu5TbrQwUMFV0AlGC3s=";
};
dontConfigure = true;
buildPhase = ''
rm -rf csrc/cutlass
ln -sf ${cutlass-flashmla} csrc/cutlass
'';
installPhase = ''
cp -rva . $out
'';
};
qutlass = pkgs.fetchFromGitHub {
name = "qutlass-source";
owner = "IST-DASLab";
repo = "qutlass";
rev = "830d2c4537c7396e14a02a46fbddd18b5d107c65";
hash = "sha256-aG4qd0vlwP+8gudfvHwhtXCFmBOJKQQTvcwahpEqC84=";
};
vllm-flash-attn = pkgs.stdenv.mkDerivation {
pname = "vllm-flash-attn";
version = "2.7.2.post1";
src = pkgs.fetchFromGitHub {
name = "flash-attention-source";
owner = "vllm-project";
repo = "flash-attention";
rev = "188be16520ceefdc625fdf71365585d2ee348fe2";
hash = "sha256-Osec+/IF3+UDtbIhDMBXzUeWJ7hDJNb5FpaVaziPSgM=";
};
patches = [
(pkgs.fetchpatch {
url = "https://github.com/Dao-AILab/flash-attention/commit/dad67c88d4b6122c69d0bed1cebded0cded71cea.patch";
hash = "sha256-JSgXWItOp5KRpFbTQj/cZk+Tqez+4mEz5kmH5EUeQN4=";
})
(pkgs.fetchpatch {
url = "https://github.com/Dao-AILab/flash-attention/commit/e26dd28e487117ee3e6bc4908682f41f31e6f83a.patch";
hash = "sha256-NkCEowXSi+tiWu74Qt+VPKKavx0H9JeteovSJKToK9A=";
})
];
dontConfigure = true;
buildPhase = ''
rm -rf csrc/cutlass
ln -sf ${cutlass} csrc/cutlass
'';
installPhase = ''
cp -rva . $out
'';
};
mergedCudaLibraries = with pkgs.cudaPackages_13; [
cuda_cudart # cuda_runtime.h, -lcudart
cuda_cccl
libcurand # curand_kernel.h
libcusparse # cusparse.h
libcusolver # cusolverDn.h
cuda_nvtx
cuda_nvrtc
# cusparselt # cusparseLt.h
libcublas
];
});
sacrebleu = prev.sacrebleu.overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
final.setuptools
];
});
sqlitedict = prev.sqlitedict.overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
final.setuptools
];
});
word2number = prev.word2number.overrideAttrs (old: {
nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [
final.setuptools
];
});
} // lib.optionalAttrs isDarwin {
# Use our pure Nix-built MLX with Metal support (macOS only)
mlx = self'.packages.mlx;
};
cuda_cccl_compat = pkgs.runCommand "cuda-cccl-compat" {} ''
mkdir -p $out/include
ln -s ${pkgs.cudaPackages_13.cuda_cccl}/include $out/include/cccl
'';
cudaToolkitRoot = pkgs.symlinkJoin {
name = "cuda-merged-exo";
paths = builtins.concatMap (p: [ (lib.getBin p) (lib.getLib p) (lib.getDev p) ]) (mergedCudaLibraries ++ [ pkgs.cudaPackages_13.cuda_nvcc cuda_cccl_compat ]);
};
in
{
# mlx-lm is a git dependency that needs setuptools
mlx-lm = prev.mlx-lm.overrideAttrs addSetupTools;
# rouge-score and sacrebleu don't declare setuptools as a build dependency
rouge-score = prev.rouge-score.overrideAttrs addSetupTools;
sacrebleu = prev.sacrebleu.overrideAttrs addSetupTools;
sqlitedict = prev.sqlitedict.overrideAttrs addSetupTools;
word2number = prev.word2number.overrideAttrs addSetupTools;
fastsafetensors = prev.fastsafetensors.overrideAttrs (old: { nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ final.setuptools final.pybind11 ]; });
torch = prev.torch.overrideAttrs (old: {
propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ torchLibs ++ [ final.typing-extensions final.numpy ];
autoPatchelfIgnoreMissingDeps = (old.autoPatchelfIgnoreMissingDeps or [ ]) ++ [ "libcuda.so.1" ];
});
torchaudio = prev.torchaudio.overrideAttrs (old:
{
buildInputs = (old.buildInputs or [ ]) ++ [
final.torch
];
preFixup = (old.preFixup or "") + ''
addAutoPatchelfSearchPath "${final.torch}"
'';
autoPatchelfIgnoreMissingDeps = (old.autoPatchelfIgnoreMissingDeps or [ ]) ++ [ "libcuda.so.1" ];
});
torchvision = prev.torchvision.overrideAttrs (old:
{
buildInputs = (old.buildInputs or [ ]) ++ [
final.torch
];
preFixup = (old.preFixup or "") + ''
addAutoPatchelfSearchPath "${final.torch}"
'';
autoPatchelfIgnoreMissingDeps = (old.autoPatchelfIgnoreMissingDeps or [ ]) ++ [ "libcuda.so.1" ];
});
xgrammar = prev.xgrammar.overrideAttrs (old: { nativeBuildInputs = (old.nativeBuildInputs or [ ]) ++ [ final.setuptools final.scikit-build-core final.packaging final.pathspec pkgs.cmake final.nanobind ];
prePatch = ''
cat cpp/nanobind/CMakeLists.txt
'';
patches = (old.patches or [ ]) ++ [ ./nanobind_cmake.patch ];
});
vllm = prev.vllm.overrideAttrs (old: {
patches = (old.patches or [ ]) ++ [ ./vllm_uv2nix_cmake.patch ];
nativeBuildInputs = with pkgs.cudaPackages_13; (old.nativeBuildInputs or [ ]) ++ [
final.setuptools
final.setuptools-scm
final.scikit-build-core
pkgs.cmake
cuda_nvcc
final.jinja2
final.wheel
final.markupsafe
pkgs.ninja
pkgs.autoAddDriverRunpath
];
buildInputs = with pkgs.cudaPackages_13; [
libcufile
cudnn
nccl
] ++ mergedCudaLibraries;
propagatedBuildInputs = (old.propagatedBuildInputs or [ ]) ++ torchLibs ++ [ final.torch ];
CUDA_HOME = "${cudaToolkitRoot}";
VLLM_CUTLASS_SRC_DIR = "${lib.getDev cutlass}";
VLLM_TARGET_DEVICE = "cuda";
TORCH_CUDA_ARCH_LIST = "12.0;12.1";
TRITON_KERNELS_SRC_DIR = "${lib.getDev triton-kernels}/python/triton_kernels/triton_kernels";
FLASH_MLA_SRC_DIR = "${lib.getDev flashmla}";
QUTLASS_SRC_DIR = "${lib.getDev qutlass}";
VLLM_FLASH_ATTN_SRC_DIR = "${lib.getDev vllm-flash-attn}";
CAFFE2_USE_CUDNN = "ON";
CAFFE2_USE_CUFILE = "ON";
CUTLASS_ENABLE_CUBLAS = "ON";
CUTLASS_NVCC_ARCHS_ENABLED = "12.1;12.1";
UV2NIX_CMAKE_FLAGS_JSON = builtins.toJSON [
"-DFETCHCONTENT_SOURCE_DIR_CUTLASS=${lib.getDev cutlass}"
"-DFLASH_MLA_SRC_DIR=${lib.getDev flashmla}"
"-DVLLM_FLASH_ATTN_SRC_DIR=${lib.getDev vllm-flash-attn}"
"-DQUTLASS_SRC_DIR=${lib.getDev qutlass}"
"-DTORCH_CUDA_ARCH_LIST=12.0;12.1"
"-DCUTLASS_NVCC_ARCHS_ENABLED=${pkgs.cudaPackages_13.flags.cmakeCudaArchitecturesString}"
"-DCUDA_HOME=${cudaToolkitRoot}"
"-DCAFFE2_USE_CUDNN=ON"
"-DCAFFE2_USE_CUFILE=ON"
"-DCUTLASS_ENABLE_CUBLAS=ON"
];
});
} // lib.optionalAttrs isDarwin {
# Use our pure Nix-built MLX with Metal support (macOS only)
mlx = self'.packages.mlx;
};
# Additional overlay for Linux-specific fixes (type checking env).
# Native wheels have shared lib dependencies we don't need at type-check time.
@@ -128,7 +321,7 @@
venvIgnoreCollisions = venvCollisionPaths;
};
exoCudaVenv = (pythonSet.mkVirtualEnv "exo-env" {
exo = lib.optionals isLinux [ "cuda" ];
exo = [ "cuda" ];
exo-pyo3-bindings = [ ];
}).overrideAttrs {
venvIgnoreCollisions = venvCollisionPaths;
python/vllm_engine/pyproject.toml
+3 -4
View File
@@ -12,14 +12,13 @@ dependencies = [
]
[tool.uv]
environments = [
"sys_platform == 'linux' and platform_machine == 'aarch64'"
]
environments = ["sys_platform == 'linux' and platform_machine == 'aarch64'"]
[tool.uv.sources]
vllm = { git = "https://github.com/hmellor/vllm.git", rev = "b99bedc737166ae5ca98cb9e3534b96e0c8c69aa" }
torch = [{ index = "pytorch-cu130", marker = "platform_machine == 'aarch64'" },
torch = [
{ index = "pytorch-cu130", marker = "platform_machine == 'aarch64'" },
{ index = "pytorch-cpu", marker = "platform_machine == 'x86_64'" },
]
python/vllm_engine/src/vllm_runner/__init__.py → python/vllm_engine/src/vllm_engine/__init__.py
View File
python/vllm_engine/src/vllm_runner/growable_cache.py → python/vllm_engine/src/vllm_engine/growable_cache.py
+1 -2
View File
@@ -339,9 +339,8 @@ def _patch_get_computed_blocks() -> None:
):
return original(self, request)
from vllm.utils.math_utils import cdiv # type: ignore[reportMissingImports]
from exo.worker.engines.vllm.vllm_generator import _build_layer_groups
from vllm.utils.math_utils import cdiv # type: ignore[reportMissingImports]
num_groups = len(self.kv_cache_config.kv_cache_groups)
null_block = self.block_pool.null_block
python/vllm_engine/src/vllm_runner/kv_cache.py → python/vllm_engine/src/vllm_engine/kv_cache.py
View File
python/vllm_engine/src/vllm_runner/prompt_format.py → python/vllm_engine/src/vllm_engine/prompt_format.py
+2 -2
View File
@@ -1,9 +1,9 @@
from exo.shared.types.common import ModelId
from exo.shared.types.text_generation import TextGenerationTaskParams
from mlx_lm.tokenizer_utils import TokenizerWrapper
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.llm_engine import LLMEngine
from exo.shared.types.common import ModelId
from exo.shared.types.text_generation import TextGenerationTaskParams
from exo.worker.engines.mlx.utils_mlx import (
apply_chat_template,
get_eos_token_ids_for_model,
python/vllm_engine/src/vllm_runner/vllm_generator.py → python/vllm_engine/src/vllm_engine/vllm_generator.py
+7 -7
View File
@@ -8,11 +8,6 @@ from collections.abc import Callable, Generator
from dataclasses import dataclass, field
import torch
from vllm.engine.arg_utils import EngineArgs
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.llm_engine import LLMEngine
from vllm.v1.kv_cache_interface import KVCacheConfig
from exo.shared.types.api import (
CompletionTokensDetails,
GenerationStats,
@@ -24,8 +19,6 @@ from exo.shared.types.memory import Memory
from exo.shared.types.tasks import TaskId
from exo.shared.types.text_generation import TextGenerationTaskParams
from exo.shared.types.worker.runner_response import GenerationResponse
from exo.worker.engines.mlx.cache import KVPrefixCache
from exo.worker.engines.mlx.utils_mlx import get_eos_token_ids_for_model
from exo.worker.engines.vllm.growable_cache import (
get_model_runner,
patch_vllm,
@@ -36,6 +29,13 @@ from exo.worker.engines.vllm.prompt_format import (
format_vllm_prompt,
make_vllm_sampling_params,
)
from vllm.engine.arg_utils import EngineArgs
from vllm.sampling_params import SamplingParams
from vllm.v1.engine.llm_engine import LLMEngine
from vllm.v1.kv_cache_interface import KVCacheConfig
from exo.worker.engines.mlx.cache import KVPrefixCache
from exo.worker.engines.mlx.utils_mlx import get_eos_token_ids_for_model
from exo.worker.runner.bootstrap import logger
from exo.worker.runner.llm_inference.tool_parsers import ToolParser, infer_tool_parser
python/vllm_uv2nix_cmake.patch
+12
View File
@@ -0,0 +1,12 @@
diff --git a/setup.py b/setup.py
index 68861fe4b..fd4738089 100644
--- a/setup.py
+++ b/setup.py
@@ -150,6 +150,7 @@ class cmake_build_ext(build_ext):
cmake_args = [
"-DCMAKE_BUILD_TYPE={}".format(cfg),
"-DVLLM_TARGET_DEVICE={}".format(VLLM_TARGET_DEVICE),
+ *json.loads(os.environ.get("UV2NIX_CMAKE_FLAGS_JSON", "[]"))
]
verbose = envs.VERBOSE
src/exo/api/tests/test_cancel_command.py
+1 -1
View File
@@ -2,11 +2,11 @@
from typing import Any
from unittest.mock import AsyncMock, MagicMock
from exo.shared.types.common import CommandId
from fastapi import FastAPI
from fastapi.testclient import TestClient
from exo.api.main import API
from exo.shared.types.common import CommandId
def _make_api() -> Any:
src/exo/api/tests/test_claude_api.py
+1 -1
View File
@@ -2,6 +2,7 @@
import pydantic
import pytest
from exo.shared.types.common import ModelId
from exo.api.adapters.claude import (
claude_request_to_text_generation,
@@ -12,7 +13,6 @@ from exo.api.types.claude_api import (
ClaudeMessagesRequest,
ClaudeTextBlock,
)
from exo.shared.types.common import ModelId
class TestFinishReasonToClaudeStopReason:
src/exo/api/tests/test_claude_tool_use.py
+3 -2
View File
@@ -4,14 +4,15 @@ import json
from collections.abc import AsyncGenerator
from typing import Any, cast
from exo.shared.types.chunks import ErrorChunk, TokenChunk, ToolCallChunk
from exo.shared.types.common import CommandId, ModelId
from exo.api.adapters.claude import (
ClaudeMessagesResponse,
collect_claude_response,
generate_claude_stream,
)
from exo.api.types import ToolCallItem
from exo.shared.types.chunks import ErrorChunk, TokenChunk, ToolCallChunk
from exo.shared.types.common import CommandId, ModelId
async def _chunks_to_stream(
src/exo/api/tests/test_openai_responses_api.py
+1 -1
View File
@@ -6,12 +6,12 @@ The responses adapter converts it to TextGenerationTaskParams for the pipeline.
import pydantic
import pytest
from exo.shared.types.common import ModelId
from exo.api.types.openai_responses import (
ResponseInputMessage,
ResponsesRequest,
)
from exo.shared.types.common import ModelId
class TestResponsesRequestValidation:
src/exo/download/coordinator.py
+3 -3
View File
@@ -2,6 +2,9 @@ from dataclasses import dataclass, field
import anyio
from anyio import current_time
from exo.shared.models.model_cards import ModelId, get_model_cards
from exo.shared.types.common import NodeId
from exo.shared.types.worker.shards import PipelineShardMetadata, ShardMetadata
from loguru import logger
from exo.download.download_utils import (
@@ -12,14 +15,12 @@ from exo.download.download_utils import (
)
from exo.download.shard_downloader import ShardDownloader
from exo.shared.constants import EXO_MODELS_DIR, EXO_MODELS_PATH
from exo.shared.models.model_cards import ModelId, get_model_cards
from exo.shared.types.commands import (
CancelDownload,
DeleteDownload,
ForwarderDownloadCommand,
StartDownload,
)
from exo.shared.types.common import NodeId
from exo.shared.types.events import (
Event,
NodeDownloadProgress,
@@ -31,7 +32,6 @@ from exo.shared.types.worker.downloads import (
DownloadPending,
DownloadProgress,
)
from exo.shared.types.worker.shards import PipelineShardMetadata, ShardMetadata
from exo.utils.channels import Receiver, Sender
from exo.utils.task_group import TaskGroup
src/exo/download/download_utils.py
+4 -4
View File
@@ -15,6 +15,10 @@ import aiofiles
import aiofiles.os as aios
import aiohttp
import certifi
from exo.shared.models.model_cards import ModelTask
from exo.shared.types.common import ModelId
from exo.shared.types.memory import Memory
from exo.shared.types.worker.shards import ShardMetadata
from huggingface_hub import (
snapshot_download, # pyright: ignore[reportUnknownVariableType]
)
@@ -31,9 +35,6 @@ from exo.download.huggingface_utils import (
get_hf_token,
)
from exo.shared.constants import EXO_MODELS_DIR, EXO_MODELS_PATH
from exo.shared.models.model_cards import ModelTask
from exo.shared.types.common import ModelId
from exo.shared.types.memory import Memory
from exo.shared.types.worker.downloads import (
DownloadProgressData,
FileListEntry,
@@ -41,7 +42,6 @@ from exo.shared.types.worker.downloads import (
RepoDownloadProgress,
RepoFileDownloadProgress,
)
from exo.shared.types.worker.shards import ShardMetadata
class HuggingFaceAuthenticationError(Exception):
src/exo/download/huggingface_utils.py
+1 -2
View File
@@ -5,9 +5,8 @@ from typing import Callable, Generator, Iterable
import aiofiles
import aiofiles.os as aios
from loguru import logger
from exo.shared.types.worker.shards import ShardMetadata
from loguru import logger
def filter_repo_objects[T](
src/exo/download/impl_shard_downloader.py
+4 -4
View File
@@ -4,15 +4,15 @@ from collections.abc import Awaitable
from pathlib import Path
from typing import AsyncIterator, Callable
from loguru import logger
from exo.download.download_utils import RepoDownloadProgress, download_shard
from exo.download.shard_downloader import ShardDownloader
from exo.shared.models.model_cards import ModelCard, ModelId, get_model_cards
from exo.shared.types.worker.shards import (
PipelineShardMetadata,
ShardMetadata,
)
from loguru import logger
from exo.download.download_utils import RepoDownloadProgress, download_shard
from exo.download.shard_downloader import ShardDownloader
def exo_shard_downloader(
src/exo/download/shard_downloader.py
+2 -1
View File
@@ -5,7 +5,6 @@ from datetime import timedelta
from pathlib import Path
from typing import AsyncIterator, Callable
from exo.download.download_utils import RepoDownloadProgress
from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
from exo.shared.types.memory import Memory
from exo.shared.types.worker.shards import (
@@ -13,6 +12,8 @@ from exo.shared.types.worker.shards import (
ShardMetadata,
)
from exo.download.download_utils import RepoDownloadProgress
# TODO: the PipelineShardMetadata getting reinstantiated is a bit messy. Should this be a classmethod?
class ShardDownloader(ABC):
src/exo/download/tests/test_download_verification.py
+2 -2
View File
@@ -9,14 +9,14 @@ from unittest.mock import AsyncMock, MagicMock, patch
import aiofiles
import aiofiles.os as aios
import pytest
from exo.shared.types.common import ModelId
from exo.shared.types.memory import Memory
from pydantic import TypeAdapter
from exo.download.download_utils import (
delete_model,
fetch_file_list_with_cache,
)
from exo.shared.types.common import ModelId
from exo.shared.types.memory import Memory
from exo.shared.types.worker.downloads import FileListEntry, RepoFileDownloadProgress
src/exo/download/tests/test_offline_mode.py
+1 -1
View File
@@ -7,13 +7,13 @@ from unittest.mock import AsyncMock, patch
import aiofiles
import aiofiles.os as aios
import pytest
from exo.shared.types.common import ModelId
from exo.download.download_utils import (
_download_file, # pyright: ignore[reportPrivateUsage]
download_file_with_retry,
fetch_file_list_with_cache,
)
from exo.shared.types.common import ModelId
from exo.shared.types.worker.downloads import FileListEntry
src/exo/download/tests/test_re_download.py
+5 -4
View File
@@ -8,21 +8,22 @@ from pathlib import Path
from typing import Callable
from unittest.mock import AsyncMock, patch
from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
from exo.shared.types.common import NodeId, SystemId
from exo.shared.types.memory import Memory
from exo.shared.types.worker.shards import PipelineShardMetadata, ShardMetadata
from exo.download.coordinator import DownloadCoordinator
from exo.download.download_utils import RepoDownloadProgress
from exo.download.impl_shard_downloader import SingletonShardDownloader
from exo.download.shard_downloader import ShardDownloader
from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
from exo.shared.types.commands import (
DeleteDownload,
ForwarderDownloadCommand,
StartDownload,
)
from exo.shared.types.common import NodeId, SystemId
from exo.shared.types.events import Event, NodeDownloadProgress
from exo.shared.types.memory import Memory
from exo.shared.types.worker.downloads import DownloadCompleted
from exo.shared.types.worker.shards import PipelineShardMetadata, ShardMetadata
from exo.utils.channels import Receiver, Sender, channel
NODE_ID = NodeId("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa")
src/exo/master/tests/test_master.py
+23 -23
View File
@@ -3,32 +3,9 @@ from typing import Sequence
import anyio
import pytest
from loguru import logger
from exo.master.main import Master
from exo.routing.router import get_node_id_keypair
from exo.shared.models.model_cards import ModelCard, ModelTask
from exo.shared.types.commands import (
CommandId,
ForwarderCommand,
ForwarderDownloadCommand,
PlaceInstance,
TextGeneration,
)
from exo.shared.types.common import ModelId, NodeId, SessionId, SystemId
from exo.shared.types.events import (
Event,
GlobalForwarderEvent,
IndexedEvent,
InstanceCreated,
LocalForwarderEvent,
NodeGatheredInfo,
TaskCreated,
)
from exo.shared.types.memory import Memory
from exo.shared.types.profiling import (
MemoryUsage,
)
from exo.shared.types.tasks import TaskStatus
from exo.shared.types.tasks import TextGeneration as TextGenerationTask
from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
@@ -38,6 +15,29 @@ from exo.shared.types.worker.instances import (
ShardAssignments,
)
from exo.shared.types.worker.shards import PipelineShardMetadata, Sharding
from loguru import logger
from exo.master.main import Master
from exo.routing.router import get_node_id_keypair
from exo.shared.types.commands import (
CommandId,
ForwarderCommand,
ForwarderDownloadCommand,
PlaceInstance,
TextGeneration,
)
from exo.shared.types.events import (
Event,
GlobalForwarderEvent,
IndexedEvent,
InstanceCreated,
LocalForwarderEvent,
NodeGatheredInfo,
TaskCreated,
)
from exo.shared.types.profiling import (
MemoryUsage,
)
from exo.utils.channels import channel
src/exo/master/tests/test_placement.py
+14 -14
View File
@@ -1,4 +1,18 @@
import pytest
from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
from exo.shared.types.common import CommandId, NodeId
from exo.shared.types.memory import Memory
from exo.shared.types.tasks import TaskId, TaskStatus, TextGeneration
from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
from exo.shared.types.worker.instances import (
Instance,
InstanceId,
InstanceMeta,
MlxJacclInstance,
MlxRingInstance,
)
from exo.shared.types.worker.runners import ShardAssignments
from exo.shared.types.worker.shards import Sharding
from exo.master.placement import (
get_transition_events,
@@ -10,30 +24,16 @@ from exo.master.tests.conftest import (
create_rdma_connection,
create_socket_connection,
)
from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
from exo.shared.topology import Topology
from exo.shared.types.commands import PlaceInstance
from exo.shared.types.common import CommandId, NodeId
from exo.shared.types.events import (
InstanceCreated,
InstanceDeleted,
TaskStatusUpdated,
)
from exo.shared.types.memory import Memory
from exo.shared.types.multiaddr import Multiaddr
from exo.shared.types.profiling import NetworkInterfaceInfo, NodeNetworkInfo
from exo.shared.types.tasks import TaskId, TaskStatus, TextGeneration
from exo.shared.types.text_generation import InputMessage, TextGenerationTaskParams
from exo.shared.types.topology import Connection, SocketConnection
from exo.shared.types.worker.instances import (
Instance,
InstanceId,
InstanceMeta,
MlxJacclInstance,
MlxRingInstance,
)
from exo.shared.types.worker.runners import ShardAssignments
from exo.shared.types.worker.shards import Sharding
@pytest.fixture
src/exo/master/tests/test_placement_utils.py
+8 -8
View File
@@ -1,4 +1,12 @@
import pytest
from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
from exo.shared.types.common import NodeId
from exo.shared.types.memory import Memory
from exo.shared.types.worker.shards import (
CfgShardMetadata,
PipelineShardMetadata,
Sharding,
)
from exo.master.placement_utils import (
allocate_layers_proportionally,
@@ -12,20 +20,12 @@ from exo.master.tests.conftest import (
create_node_memory,
create_socket_connection,
)
from exo.shared.models.model_cards import ModelCard, ModelId, ModelTask
from exo.shared.topology import Topology
from exo.shared.types.common import NodeId
from exo.shared.types.memory import Memory
from exo.shared.types.profiling import (
NetworkInterfaceInfo,
NodeNetworkInfo,
)
from exo.shared.types.topology import Connection, SocketConnection
from exo.shared.types.worker.shards import (
CfgShardMetadata,
PipelineShardMetadata,
Sharding,
)
def test_filter_cycles_by_memory():
src/exo/master/tests/test_topology.py
+1 -1
View File
@@ -1,7 +1,7 @@
import pytest
from exo.shared.types.common import NodeId
from exo.shared.topology import Topology
from exo.shared.types.common import NodeId
from exo.shared.types.multiaddr import Multiaddr
from exo.shared.types.topology import Connection, SocketConnection
src/exo/shared/tests/test_apply/test_apply_node_download.py
+3 -2
View File
@@ -1,8 +1,9 @@
from exo.shared.types.common import NodeId
from exo.shared.types.memory import Memory
from exo.shared.apply import apply_node_download_progress
from exo.shared.tests.conftest import get_pipeline_shard_metadata
from exo.shared.types.common import NodeId
from exo.shared.types.events import NodeDownloadProgress
from exo.shared.types.memory import Memory
from exo.shared.types.state import State
from exo.shared.types.worker.downloads import DownloadCompleted
from exo.worker.tests.constants import MODEL_A_ID, MODEL_B_ID
src/exo/shared/tests/test_apply/test_apply_runner_deleted.py
+2 -1
View File
@@ -1,7 +1,8 @@
from exo.shared.types.worker.runners import RunnerId, RunnerIdle, RunnerShutdown
from exo.shared.apply import apply_runner_status_updated
from exo.shared.types.events import RunnerStatusUpdated
from exo.shared.types.state import State
from exo.shared.types.worker.runners import RunnerId, RunnerIdle, RunnerShutdown
def test_apply_runner_shutdown_removes_runner():
src/exo/shared/types/worker/downloads.py
+1 -2
View File
@@ -1,12 +1,11 @@
from datetime import timedelta
from typing import Literal
from pydantic import BaseModel, ConfigDict, Field, PositiveInt
from exo.shared.types.common import NodeId
from exo.shared.types.memory import Memory
from exo.shared.types.worker.shards import ShardMetadata
from exo.utils.pydantic_ext import CamelCaseModel, TaggedModel
from pydantic import BaseModel, ConfigDict, Field, PositiveInt
class DownloadProgressData(CamelCaseModel):
src/exo/utils/tests/test_power_sampler.py
+1 -1
View File
@@ -2,9 +2,9 @@ from collections.abc import Mapping
import anyio
import pytest
from exo.shared.types.common import NodeId
from exo.api.types import PowerUsage
from exo.shared.types.common import NodeId
from exo.shared.types.profiling import SystemPerformanceProfile
from exo.utils.power_sampler import PowerSampler
src/exo/utils/tests/test_tagged.py
+1 -2
View File
@@ -1,8 +1,7 @@
import anyio
import pytest
from pydantic import BaseModel, TypeAdapter, ValidationError
from exo.utils.pydantic_ext import TaggedModel
from pydantic import BaseModel, TypeAdapter, ValidationError
def test_plain_union_prefers_first_member_when_shapes_are_identical():
tests/headless_runner.py
+10 -10
View File
@@ -2,19 +2,9 @@ import socket
from typing import Literal
import anyio
from fastapi import FastAPI
from fastapi.responses import Response, StreamingResponse
from hypercorn import Config
from hypercorn.asyncio import serve # pyright: ignore[reportUnknownVariableType]
from loguru import logger
from pydantic import BaseModel
from exo.shared.constants import EXO_MODELS_DIR
from exo.shared.models.model_cards import ModelCard, ModelId
from exo.shared.types.chunks import TokenChunk
from exo.shared.types.commands import CommandId
from exo.shared.types.common import Host, NodeId
from exo.shared.types.events import ChunkGenerated, Event, RunnerStatusUpdated
from exo.shared.types.tasks import (
ConnectToGroup,
LoadModel,
@@ -38,6 +28,16 @@ from exo.shared.types.worker.runners import (
ShardAssignments,
)
from exo.shared.types.worker.shards import PipelineShardMetadata, TensorShardMetadata
from fastapi import FastAPI
from fastapi.responses import Response, StreamingResponse
from hypercorn import Config
from hypercorn.asyncio import serve # pyright: ignore[reportUnknownVariableType]
from loguru import logger
from pydantic import BaseModel
from exo.shared.constants import EXO_MODELS_DIR
from exo.shared.types.commands import CommandId
from exo.shared.types.events import ChunkGenerated, Event, RunnerStatusUpdated
from exo.utils.channels import channel, mp_channel
from exo.utils.info_gatherer.info_gatherer import GatheredInfo, InfoGatherer
from exo.worker.runner.bootstrap import entrypoint
tmp/gen_card.py
-1
View File
@@ -9,7 +9,6 @@ Model Cards require cleanup for family & quantization data
import sys
import anyio
from exo.shared.models.model_cards import ModelCard, ModelId
uv.lock
Generated
+55 -67
View File
@@ -30,6 +30,7 @@ prerelease-mode = "allow"
members = [
"exo",
"exo-bench",
"exo-core",
"exo-pyo3-bindings",
]
@@ -792,6 +793,7 @@ dependencies = [
{ name = "aiofiles", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "aiohttp", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "exo-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "exo-pyo3-bindings", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "fastapi", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
@@ -817,13 +819,16 @@ dependencies = [
[package.optional-dependencies]
cuda = [
{ name = "vllm-runner", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "vllm-engine", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
]
mlx = [
{ name = "mlx-runner", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "mlx-engine", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
]
[package.dev-dependencies]
build = [
{ name = "nanobind", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
]
dev = [
{ name = "basedpyright", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "pyinstaller", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
@@ -838,6 +843,7 @@ requires-dist = [
{ name = "aiofiles", specifier = ">=24.1.0" },
{ name = "aiohttp", specifier = ">=3.12.14" },
{ name = "anyio", specifier = "==4.11.0" },
{ name = "exo-core", editable = "python/exo_core" },
{ name = "exo-pyo3-bindings", editable = "rust/exo_pyo3_bindings" },
{ name = "fastapi", specifier = ">=0.116.1" },
{ name = "filelock", specifier = ">=3.18.0" },
@@ -848,8 +854,8 @@ requires-dist = [
{ name = "mflux", marker = "sys_platform == 'darwin'", specifier = "==0.16.9" },
{ name = "mlx", marker = "sys_platform == 'darwin'", git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git?branch=address-rdma-gpu-locks" },
{ name = "mlx", marker = "sys_platform == 'linux'", specifier = "==0.30.6" },
{ name = "mlx-engine", marker = "extra == 'mlx'", directory = "python/mlx_engine" },
{ name = "mlx-lm", git = "https://github.com/rltakashige/mlx-lm?branch=leo%2Feval-left-padding-in-batched-rotation" },
{ name = "mlx-runner", marker = "extra == 'mlx'", directory = "python/mlx_runner" },
{ name = "msgspec", specifier = ">=0.19.0" },
{ name = "openai-harmony", specifier = ">=0.0.8" },
{ name = "psutil", specifier = ">=7.0.0" },
@@ -859,12 +865,13 @@ requires-dist = [
{ name = "tiktoken", specifier = ">=0.12.0" },
{ name = "tomlkit", specifier = ">=0.14.0" },
{ name = "types-aiofiles", specifier = ">=24.1.0.20250708" },
{ name = "vllm-runner", marker = "platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cuda'", directory = "python/vllm_runner" },
{ name = "vllm-engine", marker = "platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'cuda'", directory = "python/vllm_engine" },
{ name = "zstandard", specifier = ">=0.23.0" },
]
provides-extras = ["mlx", "cuda"]
[package.metadata.requires-dev]
build = [{ name = "nanobind" }]
dev = [
{ name = "basedpyright", specifier = ">=1.29.0" },
{ name = "pyinstaller", specifier = ">=6.17.0" },
@@ -909,6 +916,17 @@ requires-dist = [
{ name = "transformers", specifier = ">=5.0.0" },
]
[[package]]
name = "exo-core"
version = "0.1.0"
source = { editable = "python/exo_core" }
dependencies = [
{ name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
]
[package.metadata]
requires-dist = [{ name = "pydantic" }]
[[package]]
name = "exo-pyo3-bindings"
version = "0.2.1"
@@ -1056,7 +1074,7 @@ name = "fastsafetensors"
version = "0.2.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "typer", marker = "sys_platform == 'linux'" },
{ name = "typer", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/f2/c2/1504b51a4ce61cd0f6b8c6a88c0dbed22df011a2aeb99ade35e528b0e091/fastsafetensors-0.2.2.tar.gz", hash = "sha256:67c06bdbb9855070ebb036e1ce7a61a71d2ce722bbd5457534af602dd4734249", size = 52029, upload-time = "2026-02-10T14:12:34.706Z" }
wheels = [
@@ -2276,16 +2294,21 @@ name = "mlx-cuda-13"
version = "0.30.6"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" },
{ name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cublas", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "nvidia-cudnn-cu13", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "nvidia-nccl-cu13", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/08/6b/27cd4abed63ca9e4086536acde4486ea7a762dda5b36bb867ed954efe93a/mlx_cuda_13-0.30.6-py3-none-manylinux_2_35_aarch64.whl", hash = "sha256:1a5e58eaf4b1a4eb6cb0111ecb4099f6f1380403599f8992ec93dfbad8d9c6f1", size = 66904875, upload-time = "2026-02-06T02:49:19.183Z" },
{ url = "https://files.pythonhosted.org/packages/b1/ae/8864834f1c4637bd2f5da7fd8a6d2fb554a0bad16fbe53bb5ecdfcf1611f/mlx_cuda_13-0.30.6-py3-none-manylinux_2_35_x86_64.whl", hash = "sha256:35792e799bbf1498f3e0792c5c34c64bd13075de6d34d88e223de4b1000bab82", size = 69745930, upload-time = "2026-02-06T02:49:22.844Z" },
]
[[package]]
name = "mlx-engine"
version = "0.1.0"
source = { directory = "python/mlx_engine" }
[[package]]
name = "mlx-lm"
version = "0.31.0"
@@ -2302,11 +2325,6 @@ dependencies = [
{ name = "transformers", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
]
[[package]]
name = "mlx-runner"
version = "0.1.0"
source = { directory = "python/mlx_runner" }
[[package]]
name = "model-hosting-container-standards"
version = "0.1.14"
@@ -2473,6 +2491,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7e/82/69e539c4c2027f1e1697e09aaa2449243085a0edf81ae2c6341e84d769b6/multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5", size = 133477, upload-time = "2026-01-19T06:47:38.619Z" },
]
[[package]]
name = "nanobind"
version = "2.12.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/c6/5c/3b69bc3933ad3c3668ba029ad410ba8ecfdc8ee7262ff1009f3304f3c562/nanobind-2.12.0.tar.gz", hash = "sha256:0ae77c1a88f27153fa57045ee00f7b0a7b06b1cd3df942e95a34b38c5d0a5bee", size = 1002704, upload-time = "2026-02-25T09:41:54.691Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/5a/bf/1a54e3573736f3ad15fc599c5dde007937234652a1a7efd62573b4ce3a7e/nanobind-2.12.0-py3-none-any.whl", hash = "sha256:a10d3d88e691dcdf22696f9acd893fda3c5a05635763aea238829d274fcad480", size = 249512, upload-time = "2026-02-25T09:41:52.908Z" },
]
[[package]]
name = "networkx"
version = "3.6.1"
@@ -2728,7 +2755,7 @@ name = "nvidia-cudnn-cu13"
version = "9.15.1.9"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "nvidia-cublas", marker = "sys_platform == 'linux'" },
{ name = "nvidia-cublas", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/ff/93/b3c9db2c35d6183361333d2dcfea50e094c012d012c8a4d7effbfb53ef62/nvidia_cudnn_cu13-9.15.1.9-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:44cd2ec83c3ef62a7357614bd02ce7f3dac35ffcbb04ad20999e730741f0ba17", size = 415636241, upload-time = "2025-11-12T20:22:26.582Z" },
@@ -4624,7 +4651,7 @@ name = "sympy"
version = "1.14.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "mpmath", marker = "sys_platform == 'darwin' or (sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (sys_platform == 'linux' and extra == 'project-9-exo-bench') or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "mpmath", marker = "(platform_machine != 'aarch64' and sys_platform == 'linux' and extra == 'project-9-exo-bench') or (platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (sys_platform == 'linux' and extra != 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
]
sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" }
wheels = [
@@ -4784,13 +4811,13 @@ resolution-markers = [
"sys_platform == 'darwin' and extra != 'extra-3-exo-cuda' and extra != 'project-9-exo-bench'",
]
dependencies = [
{ name = "filelock", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine == 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (platform_machine == 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "fsspec", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine == 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (platform_machine == 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "jinja2", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine == 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (platform_machine == 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "networkx", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine == 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (platform_machine == 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine == 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (platform_machine == 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "sympy", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine == 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (platform_machine == 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "typing-extensions", marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine == 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (platform_machine == 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or sys_platform == 'darwin' or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "filelock", marker = "sys_platform == 'darwin' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "fsspec", marker = "sys_platform == 'darwin' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "jinja2", marker = "sys_platform == 'darwin' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "networkx", marker = "sys_platform == 'darwin' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform == 'darwin' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "sympy", marker = "sys_platform == 'darwin' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "typing-extensions", marker = "sys_platform == 'darwin' or (extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/ec/23/2c9fe0c9c27f7f6cb865abcea8a4568f29f00acaeadfc6a37f6801f84cb4/torch-2.10.0-2-cp313-none-macosx_11_0_arm64.whl", hash = "sha256:e521c9f030a3774ed770a9c011751fb47c4d12029a3d6522116e48431f2ff89e", size = 79498254, upload-time = "2026-02-10T21:44:44.095Z" },
@@ -4816,43 +4843,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/66/4d/35352043ee0eaffdeff154fad67cd4a31dbed7ff8e3be1cc4549717d6d51/torch-2.10.0-cp314-cp314t-win_amd64.whl", hash = "sha256:71283a373f0ee2c89e0f0d5f446039bdabe8dbc3c9ccf35f0f784908b0acd185", size = 113995816, upload-time = "2026-01-21T16:22:05.312Z" },
]
[[package]]
name = "torch"
version = "2.10.0+cpu"
source = { registry = "https://download.pytorch.org/whl/cpu" }
dependencies = [
{ name = "filelock", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "fsspec", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "jinja2", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "networkx", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "setuptools", version = "82.0.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "sympy", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
{ name = "typing-extensions", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" },
]
wheels = [
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313-linux_aarch64.whl", hash = "sha256:fd215f3d0f681905c5b56b0630a3d666900a37fcc3ca5b937f95275c66f9fd9c" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313-linux_s390x.whl", hash = "sha256:170a0623108055be5199370335cf9b41ba6875b3cb6f086db4aee583331a4899" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e51994492cdb76edce29da88de3672a3022f9ef0ffd90345436948d4992be2c7" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:8d316e5bf121f1eab1147e49ad0511a9d92e4c45cc357d1ab0bee440da71a095" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313-win_amd64.whl", hash = "sha256:b719da5af01b59126ac13eefd6ba3dd12d002dc0e8e79b8b365e55267a8189d3" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313-win_arm64.whl", hash = "sha256:b67d91326e4ed9eccbd6b7d84ed7ffa43f93103aa3f0b24145f3001f3b11b714" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313t-linux_aarch64.whl", hash = "sha256:5af75e5f49de21b0bdf7672bc27139bd285f9e8dbcabe2d617a2eb656514ac36" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313t-linux_s390x.whl", hash = "sha256:ba51ef01a510baf8fff576174f702c47e1aa54389a9f1fba323bb1a5003ff0bf" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:0fedcb1a77e8f2aaf7bfd21591bf6d1e0b207473268c9be16b17cb7783253969" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:106dd1930cb30a4a337366ba3f9b25318ebf940f51fd46f789281dd9e736bdc4" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp313-cp313t-win_amd64.whl", hash = "sha256:eb1bde1ce198f05c8770017de27e001d404499cf552aaaa014569eff56ca25c0" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314-linux_aarch64.whl", hash = "sha256:ea2bcc9d1fca66974a71d4bf9a502539283f35d61fcab5a799b4e120846f1e02" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314-linux_s390x.whl", hash = "sha256:f8294fd2fc6dd8f4435a891a0122307a043b14b21f0dac1bca63c85bfb59e586" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:a28fdbcfa2fbacffec81300f24dd1bed2b0ccfdbed107a823cff12bc1db070f6" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:aada8afc068add586464b2a55adb7cc9091eec55caf5320447204741cb6a0604" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314-win_amd64.whl", hash = "sha256:2adc71fe471e98a608723bfc837f7e1929885ebb912c693597711e139c1cda41" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314t-linux_aarch64.whl", hash = "sha256:9412bd37b70f5ebd1205242c4ba4cabae35a605947f2b30806d5c9b467936db9" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314t-linux_s390x.whl", hash = "sha256:e71c476517c33e7db69825a9ff46c7f47a723ec4dac5b2481cff4246d1c632be" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:23882f8d882460aca809882fc42f5e343bf07585274f929ced00177d1be1eb67" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:4fcd8b4cc2ae20f2b7749fb275349c55432393868778c2d50a08e81d5ee5591e" },
{ url = "https://download.pytorch.org/whl/cpu/torch-2.10.0%2Bcpu-cp314-cp314t-win_amd64.whl", hash = "sha256:ffc8da9a1341092d6a90cb5b1c1a33cd61abf0fb43f0cd88443c27fa372c26ae" },
]
[[package]]
name = "torch"
version = "2.10.0+cu130"
@@ -5211,15 +5201,13 @@ dependencies = [
]
[[package]]
name = "vllm-runner"
name = "vllm-engine"
version = "0.1.0"
source = { directory = "python/vllm_runner" }
source = { directory = "python/vllm_engine" }
dependencies = [
{ name = "fastsafetensors", marker = "sys_platform == 'linux'" },
{ name = "mlx-cuda-13", marker = "sys_platform == 'linux'" },
{ name = "torch", version = "2.10.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine == 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (platform_machine == 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "torch", version = "2.10.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine != 'x86_64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "torch", version = "2.10.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "(platform_machine == 'aarch64' and sys_platform == 'linux' and extra == 'extra-3-exo-cuda') or (platform_machine != 'aarch64' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench') or (sys_platform != 'linux' and extra == 'extra-3-exo-cuda' and extra == 'project-9-exo-bench')" },
{ name = "fastsafetensors", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "mlx-cuda-13", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "torch", version = "2.10.0+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
{ name = "vllm", marker = "platform_machine == 'aarch64' and sys_platform == 'linux'" },
]