refactor: invert metadata detection to exclude .safetensors

Instead of maintaining an allowlist of metadata extensions (which broke
when chat_template.jinja was missing), treat everything that isn't a
.safetensors file as metadata. More robust against new file formats.

Co-Authored-By: Claude Opus 4.6 <[email protected]>
This commit is contained in:
Alex Cheema
2026-02-17 10:05:43 -08:00
co-authored by Claude Opus 4.6
parent d0c0085f57
commit b594c2f58b
+2 -22
View File
@@ -32,19 +32,6 @@ Group = mx.distributed.Group
CHUNK_SIZE: Final[int] = 100 * 1024 * 1024 # 100 MB
# File extensions that are metadata (not weight data)
_METADATA_EXTENSIONS: Final[frozenset[str]] = frozenset(
{
".json",
".jinja", # chat_template.jinja (used by transformers for chat formatting)
".txt",
".md",
".model", # sentencepiece tokenizer.model
".py",
}
)
def _all_sum_cpu(x: mx.array, group: Group) -> mx.array:
"""all_sum on CPU stream to avoid GPU memory pressure."""
return mx.distributed.all_sum(
@@ -53,15 +40,8 @@ def _all_sum_cpu(x: mx.array, group: Group) -> mx.array:
def _is_metadata_file(filename: str) -> bool:
"""Check if a file is a metadata file (not a weight file).
Excludes safetensors index files (e.g. model.safetensors.index.json) since
they reference .safetensors shard files that won't exist on the receiver.
"""
if filename.endswith(".safetensors.index.json"):
return False
_, ext = os.path.splitext(filename)
return ext.lower() in _METADATA_EXTENSIONS
"""A metadata file is anything that isn't a weight file (.safetensors)."""
return not filename.endswith(".safetensors")
def has_weight_files(model_path: Path) -> bool: