vskiwi
fc1ae90111
## Summary DeepSeek V3.2 (`DeepseekV32ForCausalLM`) is already supported by exo's inference engine (architecture whitelisted in `model_cards.py`, DSML encoding added in #1548), but **doesn't work out of the box** due to two bugs: ### Bug 1: `warmup_inference` passes empty model ID `warmup_inference()` in `generate.py` accepts `model_id: ModelId` as a parameter but creates `TextGenerationTaskParams(model=ModelId(""), ...)` instead of using it. Since `_needs_dsml_encoding()` checks `"deepseek-v3.2" in task_params.model.lower()`, the empty string never matches → falls back to `tokenizer.apply_chat_template()` → **ValueError** because V3.2 has no Jinja chat template. **Fix:** `model=ModelId("")` → `model=model_id` (one line). ### Bug 2: `_needs_dsml_encoding` limited to tool calling `_needs_dsml_encoding()` returns `True` only when `task_params.tools` is present or tool messages exist in `chat_template_messages`. For warmup and regular chat requests without tools → `return False` → Jinja fallback → **ValueError**. Unlike V3.1 (which has a `.jinja` chat template file that transformers picks up automatically), V3.2 **has no Jinja template at all** — it uses Python-based DSML encoding for all message types. **Fix:** For V3.2, always return `True` — DSML encoding handles all message types. ### Catalog cards Added inference model cards for: - `mlx-community/DeepSeek-V3.2-8bit` - `mlx-community/DeepSeek-V3.2-4bit` Parameters taken from model `config.json` on HuggingFace, storage sizes from HF API. Capabilities include `thinking_toggle` (related: #1456). ## Notes - The model ID string matching approach (`"deepseek-v3.2" in model.lower()`) is acknowledged tech debt — see #1371 for the planned architecture-based approach. ## Test plan - [x] Start exo with DeepSeek V3.2 model → warmup should complete without crash - [x] Send a regular chat message (no tools) → should get a response - [x] Send a chat message with tools → should work as before - [x] V3.2 cards should appear in the dashboard model catalog --------- Co-authored-by: user <user@m1.note> Co-authored-by: Ryuichi Leo Takashige <leo@exolabs.net> Co-authored-by: Evan <evanev7@gmail.com>
140 lines
3.2 KiB
TOML
140 lines
3.2 KiB
TOML
[project]
|
|
name = "exo"
|
|
version = "0.3.68"
|
|
description = "Exo"
|
|
readme = "README.md"
|
|
requires-python = ">=3.13"
|
|
dependencies = [
|
|
"aiofiles>=24.1.0",
|
|
"aiohttp>=3.12.14",
|
|
"types-aiofiles>=24.1.0.20250708",
|
|
"pydantic>=2.11.7",
|
|
"fastapi>=0.116.1",
|
|
"filelock>=3.18.0",
|
|
"rustworkx>=0.17.1",
|
|
"huggingface-hub>=0.33.4",
|
|
"psutil>=7.0.0",
|
|
"loguru>=0.7.3",
|
|
"exo_pyo3_bindings", # rust bindings
|
|
"anyio==4.11.0",
|
|
"mlx; sys_platform == 'darwin'",
|
|
"mlx[cpu]==0.30.6; sys_platform == 'linux'",
|
|
"mlx-lm",
|
|
"tiktoken>=0.12.0", # required for kimi k2 tokenizer
|
|
"hypercorn>=0.18.0",
|
|
"openai-harmony>=0.0.8",
|
|
"httpx>=0.28.1",
|
|
"tomlkit>=0.14.0",
|
|
"mflux==0.17.2",
|
|
"python-multipart>=0.0.21",
|
|
"msgspec>=0.19.0",
|
|
"zstandard>=0.23.0",
|
|
]
|
|
|
|
[project.scripts]
|
|
exo = "exo.main:main"
|
|
|
|
# dependencies only required for development
|
|
[dependency-groups]
|
|
dev = [
|
|
"basedpyright>=1.29.0",
|
|
"pyinstaller>=6.17.0",
|
|
"pytest>=8.4.0",
|
|
"pytest-asyncio>=1.0.0",
|
|
"pytest-env",
|
|
"ruff>=0.11.13",
|
|
]
|
|
|
|
# mlx[cuda] requires a newer version of mlx. the ideal on linux is: default to mlx[cpu] unless[cuda] specified.
|
|
[project.optional-dependencies]
|
|
# cuda = [
|
|
# "mlx[cuda]==0.26.3",
|
|
# ]
|
|
|
|
###
|
|
# workspace configuration
|
|
###
|
|
|
|
[tool.uv.workspace]
|
|
members = ["rust/exo_pyo3_bindings", "bench"]
|
|
|
|
[tool.uv.sources]
|
|
exo_pyo3_bindings = { workspace = true }
|
|
mlx = { git = "https://github.com/rltakashige/mlx-jaccl-fix-small-recv.git", branch = "address-rdma-gpu-locks", marker = "sys_platform == 'darwin'" }
|
|
mlx-lm = { git = "https://github.com/rltakashige/mlx-lm", branch = "leo/fix-deepseek-v32-indexer" }
|
|
# Uncomment to use local mlx/mlx-lm development versions:
|
|
# mlx = { path = "/Users/Shared/mlx", editable=true }
|
|
# mlx-lm = { path = "/Users/Shared/mlx-lm", editable=true }
|
|
|
|
[build-system]
|
|
requires = ["uv_build>=0.8.9,<0.9.0"]
|
|
build-backend = "uv_build"
|
|
|
|
###
|
|
# type-checker configuration
|
|
###
|
|
|
|
[tool.basedpyright]
|
|
include = [".venv/lib/mlx", ".venv/lib/mlx_lm", "src", "bench"]
|
|
typeCheckingMode = "strict"
|
|
failOnWarnings = true
|
|
|
|
reportAny = "error"
|
|
reportUnknownVariableType = "error"
|
|
reportUnknownParameterType = "error"
|
|
reportMissingParameterType = "error"
|
|
reportMissingTypeStubs = "error"
|
|
reportInvalidCast = "error"
|
|
reportUnnecessaryCast = "error"
|
|
reportUnnecessaryTypeIgnoreComment = "error"
|
|
|
|
pythonVersion = "3.13"
|
|
pythonPlatform = "Darwin"
|
|
|
|
exclude = [
|
|
"**/.venv",
|
|
"**/venv",
|
|
"**/__pycache__",
|
|
"**/exo_scripts",
|
|
"**/.direnv",
|
|
"**/rust",
|
|
"**/.github",
|
|
]
|
|
stubPath = ".mlx_typings"
|
|
|
|
[[tool.basedpyright.executionEnvironments]]
|
|
root = "src"
|
|
|
|
###
|
|
# uv configuration
|
|
###
|
|
|
|
# supported platforms for this project
|
|
[tool.uv]
|
|
required-version = ">=0.8.6"
|
|
prerelease = "allow"
|
|
environments = ["sys_platform == 'darwin'", "sys_platform == 'linux'"]
|
|
|
|
###
|
|
# ruff configuration
|
|
###
|
|
|
|
[tool.ruff]
|
|
extend-exclude = [
|
|
"shared/protobufs/**",
|
|
"*mlx_typings/**",
|
|
"rust/exo_pyo3_bindings/**",
|
|
"bench/vendor/**",
|
|
]
|
|
|
|
[tool.ruff.lint]
|
|
extend-select = ["I", "N", "B", "A", "PIE", "SIM"]
|
|
|
|
[tool.pytest.ini_options]
|
|
pythonpath = "."
|
|
asyncio_mode = "auto"
|
|
markers = ["slow: marks tests as slow (deselected by default)"]
|
|
env = ["EXO_TESTS=1"]
|
|
addopts = "-m 'not slow' --ignore=tests/start_distributed_test.py"
|
|
filterwarnings = ["ignore:builtin type Swig:DeprecationWarning"]
|