feat(downloads): embed multiple torrent variants per model
- Add 52 torrent files (small and large variants) for all supported models - Update get_embedded_torrents() to return Vec<(variant, data)> for all variants - Remove old single-torrent API in favor of multi-variant API - Update Python bindings to return list of (variant, bytes) tuples
This commit is contained in:
@@ -1,26 +1,56 @@
|
||||
//! Embedded torrent file access
|
||||
//!
|
||||
//! Provides access to .torrent files embedded in the binary at compile time
|
||||
//! Provides access to .torrent files embedded in the binary at compile time.
|
||||
//! Each model/revision can have multiple torrent variants (e.g., "small", "large").
|
||||
|
||||
use include_dir::{Dir, include_dir};
|
||||
|
||||
/// Embedded torrent files directory
|
||||
static TORRENTS: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/torrents");
|
||||
|
||||
/// Get an embedded torrent file by model_id and revision
|
||||
/// Get all embedded torrent variants for a model_id and revision
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `model_id` - Model identifier (e.g., "mlx-community/Qwen3-30B-A3B-4bit")
|
||||
/// * `revision` - Git commit hash
|
||||
///
|
||||
/// # Returns
|
||||
/// The torrent file contents, or None if not found
|
||||
/// Vec of (variant_name, torrent_data) tuples, e.g., [("small", data), ("large", data)]
|
||||
/// Returns empty Vec if no torrents found for this model/revision.
|
||||
#[inline]
|
||||
pub fn get_embedded_torrent(model_id: &str, revision: &str) -> Option<Vec<u8>> {
|
||||
let path = format!("{}/{}.torrent", model_id, revision);
|
||||
TORRENTS
|
||||
.get_file(&path)
|
||||
.map(|file| file.contents().to_vec())
|
||||
pub fn get_embedded_torrents(model_id: &str, revision: &str) -> Vec<(String, Vec<u8>)> {
|
||||
let dir_path = format!("{model_id}");
|
||||
|
||||
let Some(model_dir) = TORRENTS.get_dir(&dir_path) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut results = Vec::new();
|
||||
let prefix = format!("{revision}.");
|
||||
let suffix = ".torrent";
|
||||
|
||||
for file in model_dir.files() {
|
||||
let Some(name) = file.path().file_name().and_then(|n| n.to_str()) else {
|
||||
continue;
|
||||
};
|
||||
|
||||
// Match files like "{revision}.small.torrent" or "{revision}.large.torrent"
|
||||
if name.starts_with(&prefix) && name.ends_with(suffix) {
|
||||
// Extract variant: "{revision}.{variant}.torrent" -> "{variant}"
|
||||
let middle = &name[prefix.len()..name.len() - suffix.len()];
|
||||
|
||||
// Skip plain "{revision}.torrent" files (wrong format)
|
||||
if middle.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
results.push((middle.to_string(), file.contents().to_vec()));
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by variant name for consistent ordering
|
||||
results.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
results
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -28,21 +58,51 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_get_embedded_torrent() {
|
||||
fn test_get_embedded_torrents() {
|
||||
// Test with the Qwen3 torrent we have
|
||||
let result = get_embedded_torrent(
|
||||
let result = get_embedded_torrents(
|
||||
"mlx-community/Qwen3-30B-A3B-4bit",
|
||||
"d388dead1515f5e085ef7a0431dd8fadf0886c57",
|
||||
);
|
||||
|
||||
assert!(result.is_some(), "Expected to find embedded torrent");
|
||||
let torrent_data = result.unwrap();
|
||||
assert!(!torrent_data.is_empty(), "Torrent data should not be empty");
|
||||
assert!(!result.is_empty(), "Expected to find embedded torrents");
|
||||
|
||||
// Should have both small and large variants
|
||||
let variants: Vec<&str> = result.iter().map(|(v, _)| v.as_str()).collect();
|
||||
assert!(
|
||||
variants.contains(&"small"),
|
||||
"Expected 'small' variant, got: {variants:?}"
|
||||
);
|
||||
assert!(
|
||||
variants.contains(&"large"),
|
||||
"Expected 'large' variant, got: {variants:?}"
|
||||
);
|
||||
|
||||
// Verify data is not empty
|
||||
for (variant, data) in &result {
|
||||
assert!(!data.is_empty(), "Torrent data for '{variant}' should not be empty");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_missing_torrent() {
|
||||
let result = get_embedded_torrent("nonexistent/model", "abc123");
|
||||
assert!(result.is_none(), "Expected None for missing torrent");
|
||||
let result = get_embedded_torrents("nonexistent/model", "abc123");
|
||||
assert!(result.is_empty(), "Expected empty Vec for missing torrent");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_variant_ordering() {
|
||||
let result = get_embedded_torrents(
|
||||
"mlx-community/Qwen3-30B-A3B-4bit",
|
||||
"d388dead1515f5e085ef7a0431dd8fadf0886c57",
|
||||
);
|
||||
|
||||
if result.len() >= 2 {
|
||||
// Verify alphabetical ordering
|
||||
let variants: Vec<&str> = result.iter().map(|(v, _)| v.as_str()).collect();
|
||||
let mut sorted = variants.clone();
|
||||
sorted.sort();
|
||||
assert_eq!(variants, sorted, "Variants should be sorted alphabetically");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,6 @@ pub mod session;
|
||||
pub mod tracker;
|
||||
|
||||
pub use bencode::AnnounceParams;
|
||||
pub use embedded::get_embedded_torrent;
|
||||
pub use embedded::get_embedded_torrents;
|
||||
pub use session::{DownloadProgress, TorrentSession};
|
||||
pub use tracker::{handle_announce, PeerInfo, TopologyData};
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+3385
File diff suppressed because one or more lines are too long
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+3496
File diff suppressed because one or more lines are too long
BIN
Binary file not shown.
+6109
File diff suppressed because one or more lines are too long
BIN
Binary file not shown.
+2715
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1519e4:pathl14:.gitattributeseed6:lengthi884e4:pathl9:README.mdeed6:lengthi1249e4:pathl19:chat_template.jinjaeed6:lengthi1848e4:pathl11:config.jsoneed6:lengthi10652e4:pathl25:configuration_deepseek.pyeed6:lengthi52e4:pathl22:generation_config.jsoneed6:lengthi221164e4:pathl28:model.safetensors.index.jsoneed6:lengthi75769e4:pathl20:modeling_deepseek.pyeed6:lengthi760e4:pathl23:special_tokens_map.jsoneed6:lengthi11330e4:pathl20:tokenization_kimi.pyeed6:lengthi2738e4:pathl21:tokenizer_config.jsoneee4:name40:91fb4f9fd1de100104925196d62b8ee06fd2ad6012:piece lengthi262144e6:pieces40:™Cït:ƒËI_Ái*xgщs|,œ4S¾çÑjŸëüS©‘|d¹e8:url-list63:https://huggingface.co/mlx-community/Kimi-K2-Instruct-4bit/raw/e
|
||||
+3043
File diff suppressed because one or more lines are too long
+1
@@ -0,0 +1 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1519e4:pathl14:.gitattributeseed6:lengthi864e4:pathl9:README.mdeed6:lengthi3442e4:pathl19:chat_template.jinjaeed6:lengthi3445e4:pathl11:config.jsoneed6:lengthi10652e4:pathl25:configuration_deepseek.pyeed6:lengthi53e4:pathl22:generation_config.jsoneed6:lengthi129766e4:pathl28:model.safetensors.index.jsoneed6:lengthi75769e4:pathl20:modeling_deepseek.pyeed6:lengthi760e4:pathl23:special_tokens_map.jsoneed6:lengthi12597e4:pathl20:tokenization_kimi.pyeed6:lengthi4047e4:pathl21:tokenizer_config.jsoneee4:name40:035a0cdd221ae0dca6b03120e20704a251a7bc9b12:piece lengthi262144e6:pieces20:Þ^Á9`›CïÜY±-L¡Ô*EC*e8:url-list58:https://huggingface.co/mlx-community/Kimi-K2-Thinking/raw/e
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1570e4:pathl14:.gitattributeseed6:lengthi16485e4:pathl9:README.mdeed6:lengthi1123e4:pathl11:config.jsoneed6:lengthi158327e4:pathl28:model.safetensors.index.jsoneed6:lengthi454e4:pathl23:special_tokens_map.jsoneed6:lengthi55425e4:pathl21:tokenizer_config.jsoneee4:name40:de2dfaf56839b7d0e834157d2401dee02726874d12:piece lengthi262144e6:pieces20:Ú*_ÉÍúºTij•Ã+‡]§Êe8:url-list69:https://huggingface.co/mlx-community/Llama-3.3-70B-Instruct-4bit/raw/e
|
||||
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1570e4:pathl14:.gitattributeseed6:lengthi16485e4:pathl9:README.mdeed6:lengthi1123e4:pathl11:config.jsoneed6:lengthi158327e4:pathl28:model.safetensors.index.jsoneed6:lengthi454e4:pathl23:special_tokens_map.jsoneed6:lengthi55425e4:pathl21:tokenizer_config.jsoneee4:name40:c5bfd839cd4cda0e5a39a97e00218d9c56e468af12:piece lengthi262144e6:pieces20:ÜŒ!ÌÙùTOã©4ÍÔÍP®_Qe8:url-list69:https://huggingface.co/mlx-community/Llama-3.3-70B-Instruct-8bit/raw/e
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+2
@@ -0,0 +1,2 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1570e4:pathl14:.gitattributeseed6:lengthi1033e4:pathl9:README.mdeed6:lengthi707e4:pathl17:added_tokens.jsoneed6:lengthi6722e4:pathl19:chat_template.jinjaeed6:lengthi1222e4:pathl11:config.jsoneed6:lengthi180e4:pathl22:generation_config.jsoneed6:lengthi1671853e4:pathl10:merges.txteed6:lengthi154390e4:pathl28:model.safetensors.index.jsoneed6:lengthi28881e4:pathl24:qwen3_xml_tool_parser.pyeed6:lengthi613e4:pathl23:special_tokens_map.jsoneed6:lengthi5405e4:pathl21:tokenizer_config.jsoneed6:lengthi2776833e4:pathl10:vocab.jsoneee4:name40:ca8dbf41071f579fbe3260f20bbe1ab896f7903112:piece lengthi262144e6:pieces360:æ3—\¾PDEËÊ‹¤¹ã·¤üÆÂÔc+Äh{"�Í_
|
||||
m 7‰�‡¶.�hä:Ù£Ùfm÷»,ðwàØnOМ˜ƒ˜"¶” ô&j�’_®Ò"FöŒ–uÖgUÕö‚¹QW½�Àá@qiiqŠÅT†×îPÐlSJƤ€\´µãR!Í=¥¢vÇóÊF¥q9§´¦¹‡Äæ¨<avÎB@µÏ ¢z
|
||||
+2491
File diff suppressed because one or more lines are too long
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi75789955e4:pathl17:model.safetensorseee4:name40:f56bc6adfb74c794203dc8ca94e0bccfe2bcd6cc12:piece lengthi16777216e6:pieces100:QM0Ts@EvšXÔ„=±6_xhÑšU4=§Ê7�j‚½üæFíM¡qµåÌm>a˜„H°*'Í5ë‘/9Bâ†^V´4H9m›ÀŠç0³^zƒ°+YS*«MÆõG´+¾.àhË5e8:url-list62:https://huggingface.co/mlx-community/SmolLM-135M-4bit/resolve/e
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1570e4:pathl14:.gitattributeseed6:lengthi845e4:pathl9:README.mdeed6:lengthi16738e4:pathl19:chat_template.jinjaeed6:lengthi50145e4:pathl11:config.jsoneed6:lengthi177e4:pathl22:generation_config.jsoneed6:lengthi100431e4:pathl28:model.safetensors.index.jsoneed6:lengthi440e4:pathl23:special_tokens_map.jsoneed6:lengthi4200e4:pathl21:tokenizer_config.jsoneee4:name40:81e5ac3ad0af6efb1298a8e8c7a10ed2990c137b12:piece lengthi262144e6:pieces20:ME�TVE@ͯÏNÕ—Ö8å»þ`e8:url-list63:https://huggingface.co/mlx-community/gpt-oss-120b-MXFP4-Q8/raw/e
|
||||
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1570e4:pathl14:.gitattributeseed6:lengthi838e4:pathl9:README.mdeed6:lengthi33998e4:pathl11:config.jsoneed6:lengthi177e4:pathl22:generation_config.jsoneed6:lengthi67046e4:pathl28:model.safetensors.index.jsoneed6:lengthi440e4:pathl23:special_tokens_map.jsoneed6:lengthi21694e4:pathl21:tokenizer_config.jsoneee4:name40:f356f2747216d7e98fee755df25987459fc1908912:piece lengthi262144e6:pieces20:ðáÌËÍ¥’……g#`¬½f¬xž�e8:url-list62:https://huggingface.co/mlx-community/gpt-oss-20b-MXFP4-Q4/raw/e
|
||||
BIN
Binary file not shown.
+6
@@ -0,0 +1,6 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1519e4:pathl14:.gitattributeseed6:lengthi956e4:pathl9:README.mdeed6:lengthi207e4:pathl17:added_tokens.jsoneed6:lengthi848e4:pathl11:config.jsoneed6:lengthi441810e4:pathl10:merges.txteed6:lengthi25864e4:pathl28:model.safetensors.index.jsoneed6:lengthi801e4:pathl23:special_tokens_map.jsoneed6:lengthi3476578e4:pathl14:tokenizer.jsoneed6:lengthi9935e4:pathl21:tokenizer_config.jsoneed6:lengthi776995e4:pathl10:vocab.jsoneee4:name40:39b35eaa97282c34db81f61a983b4b83344e10f112:piece lengthi262144e6:pieces380:õihÖ¨
|
||||
[׬-Ýâ®}̈ËUé){[ü+ü7PU�ÙnR`ÍÖgöêvHšx׬°qì¨Lz«^ÑŸÿQ@Ø‚óĈ|Å �ýÃï\å‡ehÛ¢†S ß#ŠgòY%@D:Ò©¹�¤}Þ¥XO„¢âüß§¡©¶†�œY…"ìþ|�JHïŠwÍ¢MH»*kš@R®‘1iˆ|ÿy<H·óH{�¾
|
||||
ב<Pý”@Æáè�E<²‡„Sº¯|ØÐæA
|
||||
¤Š’øó_ìñ
|
||||
;ÊRgÄÔÿ?Ĩè$ñöœ|@`Xéæ#¡¶àMÙ$n-»Ùði€
|
||||
9¼6É�@tå€j¨¥n†·•ëɃHµù,‚Ë
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+1
@@ -0,0 +1 @@
|
||||
d8:announce42:udp://tracker.opentrackr.org:1337/announce10:created by13:mktorrent 1.14:infod5:filesld6:lengthi1570e4:pathl14:.gitattributeseed6:lengthi16447e4:pathl9:README.mdeed6:lengthi970e4:pathl11:config.jsoneed6:lengthi62518e4:pathl28:model.safetensors.index.jsoneed6:lengthi454e4:pathl23:special_tokens_map.jsoneed6:lengthi55421e4:pathl21:tokenizer_config.jsoneee4:name40:8103891b028a8933068e47751bc2acc10bb59aa212:piece lengthi262144e6:pieces20:•l¶f¦7Ü.�ç²Ý a¦¾e8:url-list69:https://huggingface.co/mlx-community/llama-3.3-70b-instruct-fp16/raw/e
|
||||
@@ -151,24 +151,26 @@ fn handle_tracker_announce(
|
||||
Ok(PyBytes::new(py, &response_bytes).unbind())
|
||||
}
|
||||
|
||||
/// Get an embedded torrent file
|
||||
/// Get all embedded torrent variants for a model
|
||||
///
|
||||
/// Args:
|
||||
/// model_id: Model identifier (e.g., "mlx-community/Qwen3-30B-A3B-4bit")
|
||||
/// revision: Git commit hash
|
||||
///
|
||||
/// Returns:
|
||||
/// Torrent file contents as bytes, or None if not found
|
||||
/// List of (variant_name, torrent_data) tuples, e.g., [("small", bytes), ("large", bytes)]
|
||||
/// Returns empty list if no torrents found.
|
||||
#[pyfunction]
|
||||
fn get_embedded_torrent(
|
||||
fn get_embedded_torrents(
|
||||
py: Python<'_>,
|
||||
model_id: String,
|
||||
revision: String,
|
||||
) -> PyResult<Option<Py<PyBytes>>> {
|
||||
match downloads::get_embedded_torrent(&model_id, &revision) {
|
||||
Some(data) => Ok(Some(PyBytes::new(py, &data).unbind())),
|
||||
None => Ok(None),
|
||||
}
|
||||
) -> PyResult<Vec<(String, Py<PyBytes>)>> {
|
||||
let torrents = downloads::get_embedded_torrents(&model_id, &revision);
|
||||
Ok(torrents
|
||||
.into_iter()
|
||||
.map(|(variant, data)| (variant, PyBytes::new(py, &data).unbind()))
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Python wrapper for TorrentSession
|
||||
@@ -309,7 +311,7 @@ impl TorrentSessionHandle {
|
||||
/// Downloads submodule
|
||||
pub(crate) fn downloads_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(handle_tracker_announce, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(get_embedded_torrent, m)?)?;
|
||||
m.add_function(wrap_pyfunction!(get_embedded_torrents, m)?)?;
|
||||
m.add_class::<TorrentSessionHandle>()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user