From ba708c2ccd3c98e3558d8ccdde2e4b8c07010168 Mon Sep 17 00:00:00 2001 From: Jake Hillion Date: Mon, 15 Dec 2025 05:46:30 +0000 Subject: [PATCH] torrents! --- Cargo.lock | 1292 ++++++++++++++++- Cargo.toml | 2 + rust/downloads/Cargo.toml | 40 + rust/downloads/src/bencode.rs | 1 + rust/downloads/src/embedded.rs | 48 + rust/downloads/src/lib.rs | 19 + rust/downloads/src/progress.rs | 77 + rust/downloads/src/session.rs | 162 +++ rust/downloads/src/tracker.rs | 1 + ...ad1515f5e085ef7a0431dd8fadf0886c57.torrent | Bin 0 -> 23838 bytes rust/exo_pyo3_bindings/Cargo.toml | 1 + rust/exo_pyo3_bindings/src/downloads.rs | 327 +++++ rust/exo_pyo3_bindings/src/lib.rs | 3 + scripts/mktorrent.sh | 58 + src/exo/main.py | 16 +- src/exo/master/api.py | 64 +- src/exo/shared/types/models.py | 1 + .../worker/download/impl_shard_downloader.py | 15 +- src/exo/worker/download/torrent_downloader.py | 212 +++ 19 files changed, 2308 insertions(+), 31 deletions(-) create mode 100644 rust/downloads/Cargo.toml create mode 100644 rust/downloads/src/bencode.rs create mode 100644 rust/downloads/src/embedded.rs create mode 100644 rust/downloads/src/lib.rs create mode 100644 rust/downloads/src/progress.rs create mode 100644 rust/downloads/src/session.rs create mode 100644 rust/downloads/src/tracker.rs create mode 100644 rust/downloads/torrents/mlx-community/Qwen3-30B-A3B-4bit/d388dead1515f5e085ef7a0431dd8fadf0886c57.torrent create mode 100644 rust/exo_pyo3_bindings/src/downloads.rs create mode 100755 scripts/mktorrent.sh create mode 100644 src/exo/worker/download/torrent_downloader.py diff --git a/Cargo.lock b/Cargo.lock index c54f01d1..20bd0d0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -9,7 +15,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -192,6 +198,15 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" +[[package]] +name = "assert_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e2651f366b7ee3f97729fded1441539b49d5f39eeb05b842689e11e84501b2" +dependencies = [ + "const_panic", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -204,6 +219,19 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ec5f6c2f8bc326c994cb9e241cc257ddaba9afa8555a43cffbb5dd86efaa37" +dependencies = [ + "compression-codecs", + "compression-core", + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-io" version = "2.6.0" @@ -222,6 +250,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -270,6 +320,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.16", + "instant", + "rand 0.8.5", +] + [[package]] name = "base-x" version = "0.2.11" @@ -323,6 +384,35 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -335,6 +425,18 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake2" version = "0.10.6" @@ -350,7 +452,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -387,6 +489,17 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -472,6 +585,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link", ] @@ -520,6 +634,41 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "commoncrypto" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d056a8586ba25a1e4d61cb090900e495952c7886786fc55f909ab2f819b69007" +dependencies = [ + "commoncrypto-sys", +] + +[[package]] +name = "commoncrypto-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fed34f46747aa73dfaa578069fd8279d2818ade2b55f38f22a9401c7f4083e2" +dependencies = [ + "libc", +] + +[[package]] +name = "compression-codecs" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f7ac3e5b97fdce45e8922fb05cae2c37f7bbd63d30dd94821dacfd8f3f2bf2" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -541,6 +690,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" +[[package]] +name = "const_panic" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" +dependencies = [ + "typewit", +] + [[package]] name = "convert_case" version = "0.10.0" @@ -584,6 +742,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -636,7 +803,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -648,11 +815,23 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-hash" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a77162240fd97248d19a564a565eb563a3f592b386e4136fb300909e67dddca" +dependencies = [ + "commoncrypto", + "hex 0.3.2", + "openssl", + "winapi", +] + [[package]] name = "ctr" version = "0.9.2" @@ -735,6 +914,21 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", + "serde", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -804,6 +998,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -841,6 +1036,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -852,12 +1068,36 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "downloads" +version = "0.0.1" +dependencies = [ + "anyhow", + "derive_more", + "futures", + "futures-util", + "include_dir", + "itertools 0.14.0", + "librqbit", + "log", + "serde", + "thiserror 2.0.17", + "tokio", + "util", +] + [[package]] name = "dtoa" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "ecdsa" version = "0.16.9" @@ -912,7 +1152,7 @@ dependencies = [ "crypto-bigint", "digest", "ff", - "generic-array", + "generic-array 0.14.7", "group", "pem-rfc7468", "pkcs8", @@ -1000,6 +1240,7 @@ version = "0.0.1" dependencies = [ "delegate", "derive_more", + "downloads", "env_logger", "extend", "futures", @@ -1030,6 +1271,12 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "ff" version = "0.13.1" @@ -1052,6 +1299,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +[[package]] +name = "flate2" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1064,6 +1321,27 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1073,6 +1351,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.31" @@ -1204,6 +1488,15 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1284,6 +1577,29 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.2", + "smallvec", + "spinning_top", + "web-time", +] + [[package]] name = "group" version = "0.13.0" @@ -1307,13 +1623,19 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1331,7 +1653,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -1339,6 +1661,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -1361,6 +1688,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" + [[package]] name = "hex" version = "0.4.3" @@ -1499,12 +1832,29 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ + "base64", "bytes", "futures-channel", "futures-core", @@ -1512,7 +1862,9 @@ dependencies = [ "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2 0.6.1", "tokio", @@ -1717,6 +2069,36 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.12.1" @@ -1725,6 +2107,8 @@ checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] @@ -1742,7 +2126,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "generic-array", + "generic-array 0.14.7", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", ] [[package]] @@ -1754,6 +2147,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "intervaltree" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "270bc34e57047cab801a8c871c124d9dc7132f6473c6401f645524f4e6edd111" +dependencies = [ + "smallvec", +] + [[package]] name = "inventory" version = "0.3.21" @@ -1781,6 +2183,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-macro" version = "0.3.7" @@ -1898,6 +2310,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leaky-bucket" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a396bb213c2d09ed6c5495fd082c991b6ab39c9daf4fff59e6727f85c73e4c5" +dependencies = [ + "parking_lot", + "pin-project-lite", + "tokio", +] + [[package]] name = "libc" version = "0.2.178" @@ -2512,7 +2935,7 @@ dependencies = [ "asynchronous-codec", "bytes", "futures", - "hex", + "hex 0.4.3", "libp2p-core", "libp2p-identity", "libp2p-noise", @@ -2534,7 +2957,7 @@ dependencies = [ "bytes", "futures", "getrandom 0.2.16", - "hex", + "hex 0.4.3", "js-sys", "libp2p-core", "libp2p-identity", @@ -2621,6 +3044,232 @@ dependencies = [ "yamux 0.13.8", ] +[[package]] +name = "libredox" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" +dependencies = [ + "bitflags 2.10.0", + "libc", +] + +[[package]] +name = "librqbit" +version = "8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dadca8f521242010a4c846ef5f224c217009c92e272709cdc08ba9cdabe62983" +dependencies = [ + "anyhow", + "arc-swap", + "async-compression", + "async-stream", + "async-trait", + "backoff", + "base64", + "bincode 2.0.1", + "bitvec", + "byteorder", + "bytes", + "dashmap", + "futures", + "governor", + "hex 0.4.3", + "http", + "intervaltree", + "itertools 0.14.0", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-core", + "librqbit-dht", + "librqbit-peer-protocol", + "librqbit-sha1-wrapper", + "librqbit-tracker-comms", + "librqbit-upnp", + "memmap2", + "mime_guess", + "parking_lot", + "rand 0.9.2", + "regex", + "reqwest", + "rlimit", + "serde", + "serde_json", + "serde_urlencoded", + "serde_with", + "size_format", + "tokio", + "tokio-socks", + "tokio-stream", + "tokio-util", + "tracing", + "url", + "urlencoding", + "uuid", + "walkdir", +] + +[[package]] +name = "librqbit-bencode" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606dff526ba81e3eca33e2bb28b53afa2bc0b2c41d252333fa44e6c11abb37da" +dependencies = [ + "anyhow", + "bytes", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-sha1-wrapper", + "serde", +] + +[[package]] +name = "librqbit-buffers" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c78b907d6171a7191c162b2b60db46d254ebde6a95282b77372af556c1463" +dependencies = [ + "bytes", + "librqbit-clone-to-owned", + "serde", +] + +[[package]] +name = "librqbit-clone-to-owned" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd1e66d773ba9c475ff89286dc1d6f9d167cbb898603797467dd0ea6844c445" +dependencies = [ + "bytes", +] + +[[package]] +name = "librqbit-core" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55a02cc6fce6743ad38661ccd6fafc6cf1ae5e0106a9922836b0524dbe752378" +dependencies = [ + "anyhow", + "assert_cfg", + "bytes", + "data-encoding", + "directories", + "hex 0.4.3", + "itertools 0.14.0", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "parking_lot", + "rand 0.9.2", + "serde", + "tokio", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "librqbit-dht" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7cc129194337771a86b0399956c4d9bf1cd97c5f24d14a50be38e170f76a54b" +dependencies = [ + "anyhow", + "backoff", + "byteorder", + "bytes", + "chrono", + "dashmap", + "futures", + "hex 0.4.3", + "indexmap 2.12.1", + "leaky-bucket", + "librqbit-bencode", + "librqbit-clone-to-owned", + "librqbit-core", + "parking_lot", + "rand 0.9.2", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", +] + +[[package]] +name = "librqbit-peer-protocol" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a73129497b500505f33d1dc0426319b6a6a208f13fdfaae56224ab8c2346a773" +dependencies = [ + "anyhow", + "bincode 1.3.3", + "bitvec", + "byteorder", + "bytes", + "itertools 0.14.0", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-core", + "serde", +] + +[[package]] +name = "librqbit-sha1-wrapper" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79373a02db73159e4de7ca5d27b6eeae2d540df66c6801db2b01c5513d087524" +dependencies = [ + "assert_cfg", + "crypto-hash", +] + +[[package]] +name = "librqbit-tracker-comms" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08204944c5be677a5de8e1230e0249fce5c14abef23048e26452c6fb03f1b260" +dependencies = [ + "anyhow", + "async-stream", + "byteorder", + "futures", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-core", + "parking_lot", + "rand 0.9.2", + "reqwest", + "serde", + "tokio", + "tokio-util", + "tracing", + "url", + "urlencoding", +] + +[[package]] +name = "librqbit-upnp" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545aad6124c97201055983137e12a19f34acad565120c3cd30596cbd72e8fa86" +dependencies = [ + "anyhow", + "bstr", + "futures", + "httparse", + "network-interface", + "quick-xml", + "reqwest", + "serde", + "tokio", + "tracing", + "url", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -2705,6 +3354,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -2724,12 +3382,38 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -2815,6 +3499,23 @@ dependencies = [ "unsigned-varint 0.7.2", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndarray" version = "0.17.1" @@ -2822,7 +3523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7c9125e8f6f10c9da3aad044cc918cf8784fa34de857b1aa68038eb05a50a9" dependencies = [ "matrixmultiply", - "num-complex", + "num-complex 0.4.6", "num-integer", "num-traits", "portable-atomic", @@ -2894,6 +3595,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "network-interface" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e79101e6efcffacab279462884a7eebf65ea5f4ac2cc727b60c715a9aa04722" +dependencies = [ + "cc", + "libc", + "thiserror 2.0.17", + "winapi", +] + [[package]] name = "networking" version = "0.0.1" @@ -2941,6 +3654,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + [[package]] name = "ntapi" version = "0.4.1" @@ -2959,6 +3678,19 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-complex 0.2.4", + "num-integer", + "num-iter", + "num-rational 0.2.4", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2969,6 +3701,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -2993,6 +3735,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.2" @@ -3031,7 +3795,7 @@ checksum = "7aac2e6a6e4468ffa092ad43c39b81c79196c2bb773b8db4085f695efe3bba17" dependencies = [ "libc", "ndarray", - "num-complex", + "num-complex 0.4.6", "num-integer", "num-traits", "pyo3", @@ -3070,6 +3834,56 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "5.1.0" @@ -3231,6 +4045,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "polling" version = "3.11.0" @@ -3367,15 +4187,15 @@ dependencies = [ "bigdecimal", "either", "hashbrown 0.16.1", - "indexmap", + "indexmap 2.12.1", "indoc", "inventory", "libc", "lock_api", "memoffset", "num-bigint", - "num-complex", - "num-rational", + "num-complex 0.4.6", + "num-rational 0.4.2", "num-traits", "once_cell", "ordered-float", @@ -3480,12 +4300,12 @@ dependencies = [ "anyhow", "chrono", "either", - "indexmap", + "indexmap 2.12.1", "inventory", "itertools 0.14.0", "log", "maplit", - "num-complex", + "num-complex 0.4.6", "numpy", "ordered-float", "pyo3", @@ -3501,13 +4321,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2426ba759d848787239d80f9fdb1f223786976f87fb6c3da8188ca7c17744b28" dependencies = [ "heck", - "indexmap", + "indexmap 2.12.1", "proc-macro2", "quote", "rustpython-parser", "syn 2.0.111", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-protobuf" version = "0.8.1" @@ -3530,6 +4365,16 @@ dependencies = [ "unsigned-varint 0.8.0", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -3601,6 +4446,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -3701,6 +4552,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -3755,6 +4615,37 @@ dependencies = [ "bitflags 2.10.0", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 2.0.17", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "regex" version = "1.12.2" @@ -3784,6 +4675,45 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "reqwest" +version = "0.12.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6eff9328d40131d43bd911d42d79eb6a47312002a4daefc9e37f17e74a7701a" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -3814,6 +4744,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rlimit" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" +dependencies = [ + "libc", +] + [[package]] name = "rtnetlink" version = "0.13.1" @@ -4009,6 +4948,48 @@ dependencies = [ "cipher", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4023,12 +5004,35 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der", - "generic-array", + "generic-array 0.14.7", "pkcs8", "subtle", "zeroize", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.10.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -4102,6 +5106,49 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +dependencies = [ + "base64", + "chrono", + "hex 0.4.3", + "indexmap 1.9.3", + "indexmap 2.12.1", + "schemars 0.9.0", + "schemars 1.1.0", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4168,12 +5215,28 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "siphasher" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +[[package]] +name = "size_format" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed5f6ab2122c6dec69dca18c72fa4590a27e581ad20d44960fe74c032a0b23b" +dependencies = [ + "generic-array 0.12.4", + "num", +] + [[package]] name = "slab" version = "0.4.11" @@ -4238,6 +5301,15 @@ dependencies = [ "sha1", ] +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.7.3" @@ -4294,6 +5366,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -4365,12 +5446,31 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -4524,6 +5624,40 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-socks" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" +dependencies = [ + "either", + "futures-util", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-util" version = "0.7.17" @@ -4543,7 +5677,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ - "indexmap", + "indexmap 2.12.1", "serde_core", "serde_spanned", "toml_datetime", @@ -4576,6 +5710,45 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4655,6 +5828,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "typewit" +version = "1.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71" + [[package]] name = "uint" version = "0.10.0" @@ -4663,7 +5842,7 @@ checksum = "909988d098b2f738727b161a106cfc7cab00c539c2687a8836f8e565976fb53e" dependencies = [ "byteorder", "crunchy", - "hex", + "hex 0.4.3", "static_assertions", ] @@ -4719,6 +5898,12 @@ dependencies = [ "unic-common", ] +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -4799,6 +5984,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.7" @@ -4811,6 +6002,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4853,12 +6050,34 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4947,6 +6166,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.83" @@ -5007,6 +6239,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -5395,6 +6636,15 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x25519-dalek" version = "2.0.1" diff --git a/Cargo.toml b/Cargo.toml index e16c7b67..2daee2ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "3" members = [ "rust/networking", + "rust/downloads", "rust/exo_pyo3_bindings", "rust/system_custodian", "rust/util", @@ -25,6 +26,7 @@ opt-level = 3 [workspace.dependencies] ## Crate members as common dependencies networking = { path = "rust/networking" } +downloads = { path = "rust/downloads" } system_custodian = { path = "rust/system_custodian" } util = { path = "rust/util" } diff --git a/rust/downloads/Cargo.toml b/rust/downloads/Cargo.toml new file mode 100644 index 00000000..cc3a34b8 --- /dev/null +++ b/rust/downloads/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "downloads" +version = { workspace = true } +edition = { workspace = true } +publish = false + +[lib] +doctest = false +name = "downloads" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +# macro dependencies +derive_more = { workspace = true } + +# async +tokio = { workspace = true, features = ["full"] } +futures = { workspace = true } +futures-util = { workspace = true } + +# utility dependencies +util = { workspace = true } +thiserror = { workspace = true } +anyhow = { workspace = true } +itertools = { workspace = true } + +# tracing/logging +log = { workspace = true } + +# BitTorrent library +librqbit = "8.1.1" + +# Embed torrent files +include_dir = "0.7" + +# Serialization +serde = { version = "1.0", features = ["derive"] } diff --git a/rust/downloads/src/bencode.rs b/rust/downloads/src/bencode.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/rust/downloads/src/bencode.rs @@ -0,0 +1 @@ + diff --git a/rust/downloads/src/embedded.rs b/rust/downloads/src/embedded.rs new file mode 100644 index 00000000..c4270c8a --- /dev/null +++ b/rust/downloads/src/embedded.rs @@ -0,0 +1,48 @@ +//! Embedded torrent file access +//! +//! Provides access to .torrent files embedded in the binary at compile time + +use include_dir::{Dir, include_dir}; + +/// Embedded torrent files directory +static TORRENTS: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/torrents"); + +/// Get an embedded torrent file by model_id and revision +/// +/// # Arguments +/// * `model_id` - Model identifier (e.g., "mlx-community/Qwen3-30B-A3B-4bit") +/// * `revision` - Git commit hash +/// +/// # Returns +/// The torrent file contents, or None if not found +#[inline] +pub fn get_embedded_torrent(model_id: &str, revision: &str) -> Option> { + let path = format!("{}/{}.torrent", model_id, revision); + TORRENTS + .get_file(&path) + .map(|file| file.contents().to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_embedded_torrent() { + // Test with the Qwen3 torrent we have + let result = get_embedded_torrent( + "mlx-community/Qwen3-30B-A3B-4bit", + "d388dead1515f5e085ef7a0431dd8fadf0886c57", + ); + + assert!(result.is_some(), "Expected to find embedded torrent"); + let torrent_data = result.unwrap(); + assert!(!torrent_data.is_empty(), "Torrent data should not be empty"); + } + + #[test] + fn test_missing_torrent() { + let result = get_embedded_torrent("nonexistent/model", "abc123"); + assert!(result.is_none(), "Expected None for missing torrent"); + } +} diff --git a/rust/downloads/src/lib.rs b/rust/downloads/src/lib.rs new file mode 100644 index 00000000..4f286a1a --- /dev/null +++ b/rust/downloads/src/lib.rs @@ -0,0 +1,19 @@ +//! BitTorrent-based download system for model shards using rqbit +//! +//! This crate provides: +//! - Torrent session management via rqbit +//! - Embedded torrent file access +//! - Private tracker announce handling +//! - Selective file download based on shard layer ranges + +#![allow(clippy::missing_inline_in_public_items)] + +pub mod bencode; +pub mod embedded; +pub mod progress; +pub mod session; +pub mod tracker; + +pub use embedded::get_embedded_torrent; +pub use session::{DownloadProgress, TorrentSession}; +pub use tracker::handle_announce; diff --git a/rust/downloads/src/progress.rs b/rust/downloads/src/progress.rs new file mode 100644 index 00000000..0f9ed6e0 --- /dev/null +++ b/rust/downloads/src/progress.rs @@ -0,0 +1,77 @@ +//! Download progress tracking +//! +//! Types for tracking and reporting download progress to Python + +use std::collections::HashMap; + +/// Progress update for a torrent download +#[derive(Debug, Clone)] +pub struct DownloadProgress { + /// Total bytes to download + pub total_bytes: u64, + + /// Bytes downloaded so far + pub downloaded_bytes: u64, + + /// Number of pieces completed + pub pieces_completed: usize, + + /// Total number of pieces + pub total_pieces: usize, + + /// Number of peers connected + pub peers_connected: usize, + + /// Download speed in bytes/second + pub speed_bytes_per_sec: f64, + + /// Estimated time remaining in seconds + pub eta_seconds: Option, + + /// Per-file progress + pub files: HashMap, +} + +#[derive(Debug, Clone)] +pub struct FileProgress { + /// Total file size + pub total_bytes: u64, + + /// Bytes downloaded for this file + pub downloaded_bytes: u64, + + /// Whether the file is complete + pub complete: bool, +} + +impl DownloadProgress { + #[inline] + pub fn new(total_bytes: u64, total_pieces: usize) -> Self { + Self { + total_bytes, + downloaded_bytes: 0, + pieces_completed: 0, + total_pieces, + peers_connected: 0, + speed_bytes_per_sec: 0.0, + eta_seconds: None, + files: HashMap::new(), + } + } + + #[inline] + pub fn progress_fraction(&self) -> f64 { + if self.total_bytes == 0 { + 0.0 + } else { + #[allow(clippy::cast_precision_loss)] + let fraction = self.downloaded_bytes as f64 / self.total_bytes as f64; + fraction + } + } + + #[inline] + pub fn is_complete(&self) -> bool { + self.pieces_completed >= self.total_pieces + } +} diff --git a/rust/downloads/src/session.rs b/rust/downloads/src/session.rs new file mode 100644 index 00000000..bc1930d6 --- /dev/null +++ b/rust/downloads/src/session.rs @@ -0,0 +1,162 @@ +//! Torrent session management using rqbit +//! +//! Provides a wrapper around rqbit's Session for managing torrent downloads +//! with persistent seeding and selective file downloads. + +use anyhow::{Context, Result}; +use librqbit::{AddTorrent, AddTorrentOptions, Api, ManagedTorrentHandle, Session, SessionOptions}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::RwLock; + +/// Download progress information +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DownloadProgress { + pub downloaded_bytes: u64, + pub total_bytes: u64, + pub download_speed: f64, + pub upload_speed: f64, + pub peers_connected: usize, + pub is_finished: bool, +} + +/// Torrent session handle for managing multiple torrents +pub struct TorrentSession { + session: Arc, + api: Arc, + session_dir: PathBuf, + torrents: Arc>>, +} + +impl TorrentSession { + /// Create a new torrent session + /// + /// # Arguments + /// * `session_dir` - Directory to store session state and downloaded files + pub async fn new(session_dir: PathBuf) -> Result { + std::fs::create_dir_all(&session_dir).context("Failed to create session directory")?; + + let opts = SessionOptions { + disable_dht: false, + disable_dht_persistence: false, + dht_config: None, + persistence: true, + fastresume: true, + ..Default::default() + }; + + let session = Session::new_with_opts(session_dir.clone(), opts) + .await + .context("Failed to create rqbit session")?; + + let api = Api::new(Arc::clone(&session), None); + + Ok(Self { + session: Arc::new(session), + api: Arc::new(api), + session_dir, + torrents: Arc::new(RwLock::new(HashMap::new())), + }) + } + + /// Add a torrent from raw bytes + /// + /// # Arguments + /// * `torrent_data` - Raw .torrent file contents + /// * `save_path` - Where to save the downloaded files + /// * `file_indices` - Optional list of file indices to download (None = all files) + /// + /// # Returns + /// Info hash as hex string + pub async fn add_torrent( + &self, + torrent_data: Vec, + save_path: PathBuf, + file_indices: Option>, + ) -> Result { + let opts = AddTorrentOptions { + overwrite: false, + only_files_regex: None, + only_files: file_indices + .map(|indices| librqbit::AddTorrentOptions::only_files_from_vec(indices)), + output_folder: Some(save_path.to_string_lossy().to_string()), + ..Default::default() + }; + + let add_torrent = AddTorrent::from_bytes(torrent_data); + + let handle = self + .session + .add_torrent(add_torrent, Some(opts)) + .await + .context("Failed to add torrent")?; + + let info_hash = handle.info_hash().as_string(); + + self.torrents + .write() + .await + .insert(info_hash.clone(), handle); + + Ok(info_hash) + } + + /// Get download progress for a torrent + pub async fn get_progress(&self, info_hash: &str) -> Result { + let torrents = self.torrents.read().await; + let handle = torrents.get(info_hash).context("Torrent not found")?; + + let stats = handle.stats(); + let state = handle.state(); + + Ok(DownloadProgress { + downloaded_bytes: stats.downloaded_bytes, + total_bytes: stats.total_bytes, + download_speed: stats.download_speed, + upload_speed: stats.upload_speed, + peers_connected: stats.peers_connected, + is_finished: state.is_finished(), + }) + } + + /// Wait until torrent download is completed + pub async fn wait_until_completed(&self, info_hash: &str) -> Result<()> { + let torrents = self.torrents.read().await; + let handle = torrents.get(info_hash).context("Torrent not found")?; + + handle + .wait_until_completed() + .await + .context("Failed to wait for completion")?; + + Ok(()) + } + + /// Enable seeding for a completed torrent + /// + /// Note: rqbit seeds by default after completion, this is a no-op + /// but kept for API compatibility + pub async fn enable_seeding(&self, _info_hash: &str) -> Result<()> { + // rqbit automatically seeds after download completion + // This is kept for API compatibility + Ok(()) + } + + /// Remove a torrent from the session + pub async fn remove_torrent(&self, info_hash: &str) -> Result<()> { + let mut torrents = self.torrents.write().await; + + if let Some(handle) = torrents.remove(info_hash) { + drop(handle); + } + + Ok(()) + } + + /// Get list of all torrent info hashes in the session + pub async fn list_torrents(&self) -> Vec { + self.torrents.read().await.keys().cloned().collect() + } +} diff --git a/rust/downloads/src/tracker.rs b/rust/downloads/src/tracker.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/rust/downloads/src/tracker.rs @@ -0,0 +1 @@ + diff --git a/rust/downloads/torrents/mlx-community/Qwen3-30B-A3B-4bit/d388dead1515f5e085ef7a0431dd8fadf0886c57.torrent b/rust/downloads/torrents/mlx-community/Qwen3-30B-A3B-4bit/d388dead1515f5e085ef7a0431dd8fadf0886c57.torrent new file mode 100644 index 0000000000000000000000000000000000000000..86d6545e84c47a5ef160cedb461e26e1b9bddd8a GIT binary patch literal 23838 zcmcG#W0Y)bmZ+V!Z98|`wr%H5+tyCowrx9i+O}=mzUTJ$Rn@6`t8Vugr#pVdh%w)1 zJ?q7sbB%~FVxZ?VbTHO;GBzSGaA#oRw6=7zb#O4YaUx)#Wnktsw=uCbV&OC~w=#CL zGGgVlGPW^wGBal|X6CfhcQUhL<)k&#H#9S5VBj=!HZ?W3F*VUQH2wohYijOfZfaxe zU~Fvsk8$+>?{Vy$Mz*dtR<`;^3>^PK(06ijFgI{^GIpf3vNg2)Gm3%!KauzcjFIX8 zgjpLq=^N=g>Hit{Uw32XR1^^qmJy+~Hu@`C4D5g6{3RnsMoxVrBV(hVPPUfDHjcCw zjBV#KX`afO_G`1%H@rIe!QQySa>5s*=b@&%&fsy6Eda#%o{&W0)pFiXO ztpC5k|M&Sb{m=UU8~lHtKlA^r|G&ZiHUB?q(Er~5|Fq%FZH$cF{&MR0+tK=W=)dI6 z$j<3#XKZM$Z}tCY;h)y}c7I)&@!wMalIpyCRxredCzmfc}=afI{>;D1!OKyKu zpTFaN{yEqG-z~s@xn?ub{~P9CGG$=lbg?zmH~2R*_+Rf({#cGbf&P^f0}I>VZ}yMA z?0>gc|7ec>+{UnTu>GwTaQvgc_;Ums{of-P*#7TA|L0H!29AGRmEq5Y_n)Fz*%;Va znErOzKf1qv6vIEqu`n@ku+XzHv9kUxl!@^_tw#S0U}a`yV`pXkyVO4c{&hL{CjcWG zGaCmz6Z_v~{|WG~^YNbn42%pc?DWixEFAyx7XEcY{3pO4Q^4}~`Trwn{J)ajp9ius z{N4GF^Y(92&7TJ_FfcPRva&G$P4tgl^Y>%*9~<`1!5nP#e_s?vhJOV9`TYGko{^1> zm7VGDMt>~%Klbk*9X~UtjlQ)pGd-sf6Fa++@gMy>3j>P@i!nVri?IotK0PxNgOL%t ziN28uJv%$AAqyJ=Bd49Yv7s@+-ySen+1S_^8CZ>3|G&>28R=Q*={cD{T3pPlpg*IC z#96U=bx&Rk$YMdY(&#I^^+{U`RnE7@f~$Yu-7)I46$bG*UY)@MQ5|lDzj6XSKI$pBV_(F3fHE){)p|SyjPc?)C+<2KxblLL zU3z!^5(k@oXgUZB*$bYs5Pxf8h?)rC%D`578Y^!}97mF+7%u5m#dZAhhJAky@? z4X{0P_!1?KVIHxeb8}62JxrlU^aL8NT=<$$5r$dyjF(Tu7YevPe0X{GR|)(T+~#VI zsLvSPDB)>hc>26x;uRpLF$^)FqO!O&){0<$tnDojFnO8to(QZBY@-fJgqG~H&1;WI z8&6wVXcInV)@1lJ`#Tld(ksVYyPf`}70?T?CZxaRdYT-VO5s;R84!Cbmi!GvpUkjy zeb33}4N;zNVj$%#oHVGYb9m)j!9!uXbwM(OvbB&QAfmtUjqc9eIHZH*LGFQ|-o{Su|&_6y!VOnM<)!PjeFRE~s#y_=HPQO?a57&a|O{0^n8 z#cw#!c3Ft}tDhnQ>riDj91QeBIh>&`zqF!0q;j>Ya~xh*Cv{c=)mVV`z)a}dDhK^- z7D9G!t`0zX$c|;xN|1Ws9EGbcFT*6_oHEYiHCEqrC4Meqk0=VE857|Yvdol`!7Bpr->4u$XAW!g0$9ozCjF>5K!FkJn$3pAp?$oe7KF03 zSYv}m%>Lkrxj*O_Fp9%qxIrBOicbzP z)ghrL5cqwm2Clz}g3MGLSg`FkjsY=hTUt&Clh|O^g~Kg?t7r)@d9MZ(UxKb7;jb9695&I4 zkC1niCUFKtB1NgloW|fdqrK14h3|T6s3Y9rAK_?{C0!MgsrvIrm|+yo%(Mw-uotY-WnVhpQhlYBA4?>e>pL4uS+SlXv#P*BLcuNwbe4;07W|tg{E+ zR)02dp+8httgaFh%&74Z2G?%h*1-nGZDr&CG(P5PKdx9C1_yx{A{~FZFO`;-wfiDbcXYnW#mrAkRRN(nY1&5UGQ(e>4+{m^#s zG?&&&L|!oFr|Z`=5L_Z#`QA}n^>?W=tOOPRCKL0DjB$+|`xpEa*pa?u4z^G^dY)VC zw$x-5?T>f$1X|<5X41qGsv*q2f-B2*Aw6{)7i>~{{#AsjN48>jU&EeJ{MfcrID{eV zQ;m{Y(G%JDc+KP=FlC>C{qqfyHG|k04VzMd5v@`4u zbLM6XD=_7j*I9o|X?qyeo@a)hvnZje=N}2kMYw?b>CyC<^+q$Xa3aT=h}$@4i*ArZ zQQf#q>(@>=o7uA0>k#JAkbjoVm%HB;LN(tT%I9#n?p7cQPSh0k(HEIB&^8$qaOlL~ zKs>zkA+Y3D*Oo~ZMy3=^uX9ioU_^}ekb*{%_zItEy@B08qhI`P;Jw2U$+%wrUgklX zktcgg*|#A|#J=M`P%W@C`b0{aF(00m@ae1qq|~>?t+zX^)&K`JP|Bn~+Be#~wrrqd z-uKM$ZFmp;7&VTjxzA*pX$g?64|0_cn%X;MDCFEP3QU32xj@B!EV`^GoLryC*qxB- zT@k({>nrFsUgyywI@$7H=E0+<2;h(kKghOhi=o}h4 zZb*iDWvL(w&vpJr4@hf!2LCN(Z4hPS)nqoeM;gXdko$O?7;H?x&IjLQBhJAF0O{l_ zzcpyE$W1O%(-wTO9}fyfJS;ESk%ae9Cx8dKEvOV~xfXqK>$y1-ISRk$VUy9t{~<;x z5{hx~CP`jG_U@sFrRTD;a3puakUZTUyA>E_L5M1Tq`Y0SCcryyRGV{MXx2v&?mV3} z)#C_iu&S?ku0^crR1U9kMW|0Z!KDIg2U+pbP>PMDTX8xq9OQu z@kjuz^VpW6!zl-jwwwu!k{(PejuA@eE#z~jbVsr$$DkBcqfgTV_DIdjK}XTanQ$5g zPluNwd00*R7T2F7b94a|Bu}RG@T-6m#X$GoHZevE8n?8>Y`o)=@cKpD%Y`&HmYL4G z{qbmJK}Y{OQiJbD6rM6V5{&om<0y?QRewI3vjtyDG2(aER{=O_E`%R|n%GFB>m=!R!5^ zpXeajzb-FU!q7kqXsZ>11 zg6=q&y82OQFkrwyd%pM`dJww!1Ji8sK@$u>*7Cf3w`AH*=D2u|o3$gG8C^HmlYnfN zV+JQ2cy_2=vbj>Z;z(RU7O_{*ub5!Ldkc8;hK;s!y`NZ#EEi7jA>`fVFsh6?y_wO;mO~~E z;qsn@2@&QvAOB27kJo)#4=OBBa+-M~(ve8@iBOFh{Wt{ul0P0MSItr@5jaVFwFV}M z!kt%*(72b0r;@Yv+jC)w$cU>0Bn=f`?~WJe?#GCu3uP_?T)%8pc0dXdSbc6v-a9K+ zGB5&1iO0oBW>IhH+nHi*QF6nPu%5>Ow7?rkTBRH)&H)du-`SeuPg?LI_5}wtG1Nby z#{7oJ7m;c;$Q~a@Ok=nen^7U62z=_htSPXekDDUdM&CBUx->3J-$@%_Y;f%$RDnWU zg<-B1PLRHsu*c`W)W7Ih5wJ&jnB+MtT1SwHePXYscKV?wsTvG08f72Fl8?ZOcL2n9 zVqBUrRMJoCZq$tsu|cPvOtp})Hy6M`a*rChbzOilkV^9 zuS73R$=Uy_?h)$>I9%@8yHS2GOB%HIw~*$n#!TZoY6q%Q&$UXP-n>8?f4N|JNO!>V zUA)0D0UX>F8)V71V*vjRx{(k?Io`;tx#m3aqG}Hla4?SgE7sU!p@N=}&YiL9_8O<^ z+}F#|wdqLeo^W1`LrRycnuyiL^j)>t01HVkxu;41hZ-`b??OxJ5tm1_D-FT^{H6js zGJF9%v)j*4jVmJ%iqgiJ$^60YL30B7pV6{*7!}D7|aAxo0`+F+D$(%6%9p_ZTxw+pw*+LowRh)+%Bb>>!Y-ot67B6mDdP z#mNc_wt_j*7dl-mYqB)-04X8(qIWqg=QzmF7Zx6=#Jmr%bNCS+kkrHRc z5>XldCiZ@_Y#^ufJI3Ou6QLDTmuPz~VOC$1fOk}w%K%v)K1Z$z2fbrv^h^cGcd}b9 z=xC0*o{5jXrWUM+`b}W!s?gi4lNJW_rkC=!N33*7gb!`bE zKK$ZMxTj7`v31l&C>z3N>TOn2iC=tkSh^mdr||@i`$863B>9OW!tJ*~8s(U)S{l)< z%>{9w=8W$S-NZN_kE}!}h#o1I?q>DyW#wZ&s@eQl?BLN4&USa>ES22u7ynvIK)(JU zb2h{uzeq}^?vie}tPnOJEucbFI_$3BNPP`f_sM>|LGw6mwJr<9{b;@h=>3SSv|}-} zQ|9W8ePIX3%1i*@fsPL_B%?2%8jIYWN}JpYs1v6zzjFBEtBXunV&hprVLw9|mXco0 zPtiH&ZZjKwBJ8GLzL_*r0>b(>pwj+SncpPlAH@6BD@bxQ&)`$yCQ2a`q<|hKJjxsF z-l8iMYurE4?c;a8Gs7>mCa)gy3@gXik^g%2+v^lV+z5j+Y)97!?EXXZ7m=>VxVl5e zJg83f@~nO{Mo_fS&aCXLx~Y?!)YZO-l{lPG!C*?S)`tkD@|eftQG!KC-w(sh2>MBW zZD#Z?H8lf23nI2mwe-8=mWFp77kcsN5H%sC4YiTm?K4}# zr*dX;&3GhevU7$>Nb=l7nQj#a8#KcDH-*9ERzq-E@Ro2R2By*jJy$T^gni_vM^D8j zts}w#%0?|-ohZm8hV+)Tln*naK-*eMN^vAcY9S&6)S#lVe)Tz9Tm|WxEzY30>q>k7 zfp}kYD4uEwd2^Dv-BB0Grf++iSk1K^4zD^6kRiE>RE7fYv}AJ84SB35*KvrYKE`Qr z%$SZ141&rh*6O-}p-By8J#Otn{!yF)?=xgIN$N-F%Jl-JraKOO#WOyg33vQ!Rs@gX|2B@F<5qE@pvxHXeU@j4p zJp<(M)fqgq#9Du0?yaqg+4>k+j5IEIGj$v$-y@MKve&X;{SUCYKPH8jt!aTS=alvdc zq<>gtR2N#8TE2IOJ{X5QhCp4#@%y!x3lJOY#I*N_XmPb*bmT*}rmk(L4qaHMWY@A+ z#-NcTqu{l5N%VfhbU;9 zz^| zFlK1;xg0GExR!cfkbMA+dY0A;LL2^qCd4#{n!b;u-ykILT1^jh1&ge>nax3+^VqZk zEw(L7J()FW(+uVxSh_le9Ctgl)K(7)8z@1iW~aEFq9lSRMF>K^TFn-NdwDz{>(#SZ_|4lA(v=z z%Px~K6=U8C;Y^T>mKDxTA4bF`O2!OqtMIYYUhW8#c1-HYM4mTygp0~;R=R+ryrAX_ z{P5u+d|T|JFr~)@^fjRvCL4&-tMOH_+_G210D}10L6{ySKM7J%Hx-BX_8Sp{eD~+{ zNAMcuZ-Hyofy=K}9y+CN*)KIvd_lxQyxT|ai_q3}2e|p2+hQh>)9DU9@-+>+csJcPFW3(=#{& zwj0~yS7}VEA35Yr*^TcxP6+VAY5WLMx#>^5Hi&k@{)E zJv4XA`Wv`5TI>u;Xz)+cN65>%tKpn(obY1cBRRe%y_V|J3>au4dAE03{sq3KNgR(M zW+3jOu&UOPEF1ApAPL+#+ym->R00`MbeuRaQwC`C_QfUdY&Kv~=4B`&I{@pwKrarT zPnZ3O{a%p#7WVVGxF8rX-ug5tEeGK<7fOS3w3}funueB(Luc9_oY>*mpRfRR^pyel^@ZmuV)0a=P=$jKtfs*-xoCRkx51n8P?s zxWqH`?)v?b!KUtK8`4 zqSd;ITKTJeYn6#ZA545;Z}VUFyw`6**mhLHs9ZT8P=r0Pl+c%F0e#S3~ZyIH@sMag4o9g)XaGqq`XwVFhV?#%e2{kS*<0hsmu3 z>puc=ozc8uRC!J~^?-n4FWyN%Hi=BI7OU)fMuD??5NHZIo0sr723r&}trZNk6emxE zAYLh<)b-PTuyGTtlJaU!;ZI@I4cfBoS8avK5xp(h<6g8JQ;WCjRw2wQPovjwmRX03 znqDUZR&TdD)<#F^SWasKBTg0o3&Jgu0f$dVp;a{0zgiB)xtq+4G-3IIOgh6TGOZ_T z4Y-x{KI6p@Sh&=XcJJmWxw1?J8zt6rZLroq1Zq1Pg2;W0B{J(@gtUQzztd?feeAeB zAxF11pTJPHk{e$R}KX!JC7uDOf@k zRtwKXUruNLddbssjk=uHjRW4$7-gSX%h*#&+C8|!T`6|G^}ePls8a&eA=83?0|j%d zT@>R%Qo@V9fOhHAfd(dQw8ICr1(}i-cLKg?3OIx_W>3`klp}SZBanhR2<&l5^y?s( zpL6xDuY>7mL+t$rG{(McKm_0KZ>C0xm^=+gUCIQ=%GPK~h|-+6nG$xgp^*!q<1pk2 zJdE^C*9DwfBK$@MV*LaUX7Nn59GSdndpHV!y%`vUpfht^u9+>pTm5F*U_ykUnicC^ zoY(XR&|iH>iA#*EZ`wH09JQnVMRn`H6$qtlZK+JXYKHyX6ND{+T>!Ohi+*VAULECY zV8fInqKd+$m8e+4nbs=iU)Fw7g{rf`V<%PNN1X4Fr2N>!Jd?{$xK^X0m+v3e!2YpV zlq=;lZ6kH=gOd*}TgyL)lVE5VVu2N-=+nL6qqc|WSnZQYA8RGxlfg`;#RcO$&Yc_M zt2(J+bA(5A{4=HpRvyY2ix6(OwS_qcl_b0>KE+qP z=*FQJtKcz!3kDg|aFW=bR%+J=h)L%Rv$1UK;~A;pVYf&-FHyHNxrDte^o6ywkFpCY zFoOega8o_lNZ^Q|*pmJws*&=2U)H>Bm#=Nyjufj4A4ujIcH%;tpn_=KSevxgLHEY` z4Z(9!>mdlf8EkiQpIUSfhy(>1FFd+4vaV#MiM;(VOIwy$`ygXK#US)kUO!5u$t9g3 zd}65jkwcXOWuxNMYJ;^Zy=N#wSq(4^|6%yxA^Cw(7s@UM1j2E}W8boxV{}rVYc3w) z2e+h9lj$8WWd(En2--yN$y(%vfTdRV=ZOYht0(Wuk@}po5wfF-2YBhWEOKiTT_fJR zqd%%aK9Q?dA5P!5-nyCoWgOBTL}j{>LrFh?i0bJmI zKN$qz=5=*r(*X>;Sl1y1y-&W+brr_=wZ`{#*Etg1y4ed*9nHovW~%@C=>~y~f+5S5 zQm8Ekn|T0MH%TkKIT@xl{c*39VeK2)6*gS0?DkEu3t!`SCstRp4{;65P7oLI#lsHE z*B0wmw4fm0nEt7-c}`gd6@Yi7W1IJ*GWFzbP?45(j&N`CfJk;t9l)|W8VELmraf2% zD=}|p1W-Up9}X@s;?-ZgRqpmgPgqt4?1p^iD8&iud?gl&eHeQAgkQT8l;(oo=68P- zRrqB~f;2z&V|(r^wUd3d-WenLWTRw+Hyq9TjZ&zvCEF;{j#X}yxc}FX=|kdNJ7xn0 zgZbq*-n(s_bqb`ql|;I+XA~rUFyHFlAm2Umz34qhjqo?5>5eK4spnM4t;0 ze(+V9<}g>9b8LQ&gc8o_FaOu?=;0#E`)Oi2KxQRHYQsH8ce8t4>CmaSC|9%Z5XD)59*LE- zOgftd6%J*wZpr|GJRrm&?8}FrH1Y5LffT3@SUkI`N!{(I8Rm1cq04cVFwSgi3xKYN zlKO2f#(9R%GPn(_%Koc^%eGlSYUV0j6qazPV#dt zhK7QHjr~%+*O@-3fCpeS-}~!v+reLil`t%mo|1m!g>6$KiF42rj^%hBd5w^uH45Q0 zLmA26fn}J67~Qr@^>QC-4}{wKJKwQ7F&;qKPRS->{5wArKDCtBP%m1z4Jd#$h7^ift5RfA@ave$Vx)B zbswkGSZu|XcaY2@@h}i$J?^68COvP#(-`b;$-}*X+*>-CFq=x1x=eyQctivaCb0)c zT$aq15;E^UQiVH)B%+53A*{~g*@^LRnN{>oEYq|ef!$jq?0SSZ%mfc|;#LksnGLE& z#a^!5SMzlZwqogJ9F9BP!@7MDG{l%|I;hMW&OCB8%=-KUwn=LtkOcJH#S^9pSn%#q z^VtQ302K(cHF_WN^+z%m-|SX4j6BA6%?at<_+BZ6hVgJ1(xB)L>?f>VyG5;cXeGPd zyrpUU6>ip7mHpoj>Q6q`vW7SyC2rR-br%ToD)%4K=9~_%ag^35!&(x3jh`+e#_S+F z8OHsy5Y=QuAUSqBveQCQS&9fsWhD}XeMMdFNU%KK^x}F146-+6J+XeM9s82@JEhNJ zsPOrG-!nw#zP@yYFk2A%OEjxQ09aq<@K<;Zk=~$}D!}Dri=!NXn*FH3v#VD zCuX#T$3?*;#t&_ZCr&{+RTsdMoSQs!jqCB+OxJTeW$iQeE5;XPZ)bEeV2IiTdmg0$ zuY@)+4(W@eE4rXXB)f*_*(XPx)bjBn!nhWn15aKsvt1)7ySwj;r0B%N_i;e9E0lQ@ z!}%1}=GbKnun3g1x1S*1vzqo2V|miQs>`UHXucz?Rht7Y<~Ac7X$p^31c659Sy|0? z^7%1h!prQ~iW&E7+sp<|4%9a)ZJ-4Y)DfKJ-hdM}Z5sMU!ON@mh>Tdj+u!SkeKEE* z8ne`-D==u~Ff)q&QtF9TIUpWac*}Uw5Xa%? z2RkBI_<8}^5jPOdh{Tqf3fG$3A4c(FDMQVIZxqCWmnt}4-fj<}9u|06@3s2Slof*L zBbnI;R+q!jEHn+#bg8f&8G}>h31I8okV>?f5DCxGOR)7*1EF}O0R14FJBt+3m2YP} zs|)g53mANc@PBbqYt;l35TrpeC?G6#*H8uchs6(wSfUw8thcAI%o@%{|>CHFbV&qDeggzd7$fPM@wr zcfiyx5=AJ|aBY$ylhat_S7G;0Cs!v96qR0<$lry14bv#!g^>iEJ-fW|1k_wzLe6v! z!+MP&J0a>0Sw`Iu!ittQ^E*AY-oGE4;6V!_l7_d+hD^Tm5t4=+#8z@Y7gh9^%h4K# z3UL=nl5ObRl;c||q+IC)($I+&9aG7M*7|976bgsID`mrP0Q-LmHz_mfKrTZ>A8P1+ zf+^jZ#_q|WSqD}IZUI^0>Lli{Jl@fG*#|a3Uj})Ub}W%bKg~Y}o3Uion$j1aRqg0M zgnWy{fJw|1^bKUI4hp#*ywu)(ZMax6VKNrGMHIc|yNZzf=&jIy-m;#_b>H8H3TilX zy?}?3P|fIa0?DhfQO#&T3!O>M2l1~W3VLo+N_%Y(+@C`=1i5W`Q8Sr_toO6Otd*!u zgsZ=XwO}Hyd~{7aUTwy-l&Gs_k0%) zZ>N3djD7DY1maVj%GdNAEO}-t!zaMVP^-|`|y-;3LVYG?C* zvyx^Zj_qc@I~dkraoB}UI*gf^Pb?`|i2wkr+qmlV)?ner=!D^JT(>J%pOW0Y{LX%k z@6`>wVmAHiVbtWvuv@~M(canKdhlEYbniR7QeZ+8pa&ne%l<;>J4ttI>9STjCG}CH zD-`gEIvsgEzB4Ken|sZ_dmQREJ2SyX(MfybDKl(@zDyBcB~BzL_zaiy>!MV6s-m-V4jW-NLByBU#~(@v zHVlF3I=N`P<iM)ug_V;Lu853+xtp8b_eRf7tTzjW+OdTJNd<4gqzP`dF;e1NdOIE=(9^LMl{Z-FS|RP&xE|1v_y z_lf`nr011gLCdY1Z%y<|(0xVYIaq=$7$dd7IBDn3!*jjz(dkN_wPoxaW1h*aio!|V z)-~^j#YX+_>go2fR0ss~BnwcdlENmW+S?8((; zPS|c5_Dp5RRp~6T_0(ZM=mKtf`Kql`BVR=Id?&_(KW!+m=Iy=?-tA73ob40~Mdy-w zDOp5?*4uQJDdSOuRh_ktJA?!V*^QJ9Z>N{bi8BEg7w=PG{XyP-+h2OcmbK#OKsQZH zQ(?37=R0!bZ$<<2))>ehv5(M1ZD|k2#;`Byp#0z{#c>vlp8Ac;Y@Q=(Xd+$9eB!v~ zjyt#wrhPKs`>^8UK8?=%uwyYcJrrF5a(gnUfF)Rceei{ICleuw=ttF2dLZXMG9d;o z8LDQw>=h9s1IKSo8DtGeIy91IP6a3mWR3t|~d zcXA}3fUkJuhFv7SR#f}O3dMT$Ddh(Fm_|qnp_fxeVcI(sjIe|pL$Pr4%FBPrGToXN zs=BZZ7qK>cs`}!xLjX}jK`aByoO6*KzPvTS0wHT#m;+4BnTQF49L!lz71dR|iNMnXhbd}VZTP=TP5 zEs%R@-y9pdbOF#1qQ5#pDW2I?F$`Cp>6Pm@!ZirVcXs6uHJO@YE44YY63pn^Hoc6ihP2?>a>B6K4=nVi0-CVHO`sYNu{)|X9}&bOrDoMPHQOx`wx?{wE&QfAgM^* zzXO7sPOg*f!iAKnIHb@U7g__j0gevBoO@ab#8L#aWVtUxAofln!b3j$nu zu}|?-%(1$h?inzbS4&IUdm5iVP%N+a#Eq~<&J2T)Lh?Lf9a%)SB85aX-0tuDK`x|W zKeq=a+@Wj5g^w3K)z0P&aYGLe(VOK|`aiJAOs%QGK5FSwgTuwS4-Bjw9BY*8rmc3&Q=(MLZ0z4ccJ2WHDA*1}?Imr#EB)59I^k$Rw_@2x-VD zA8f379U`)at$81P428US8B4AJx~M52Ak>+}P68?7aqEb4A4bywHqm3DSUs#N#ZHP+ z<+?Kx?9iH-_ep>S@5yn}Xk?1^D_&>g_|cFmqx}N0JMr{HLtqiW)!%=#nYpWLf45G* z2`+8b0w6|oSSd;?AaqG=el|BhUd?FyiIff1iw{r^YLx?6mpp>@!i7en)eh}6zzsp0)>uczcj5`x@!YFYk0|>a zRo0a(+b@b!2Tyu=>Aw#s-pXy2?1F}uomP_F9^G?&WP3jS8XPpO?&@wp6!MLmLmUMbDZDwWjQ zhYTc$R04GGBd~f_`mrob+N=%kvYEbaokA0I_)7#Y*Wcdqb z_2bI(u{^a-9W+$|NgA|oG8(m%8&5ArE+++ua|4Xf*TiN;(HN?i!{+Ec37K>ICv zyyG?BjC@IwdbGb0I7Lxa!>5Xns;Z%&Ns6H9cUxc>VWhrsRs-2R= zX9P9We_pZr`G1aak*$2|5j1skW%u>$CpW4WxTxPIq85I10_Q)nll+=-UvdVH0iP(5 zo+cCVJ$KKX-ta*^kZ;!7jC?d)Ru}Mi^@P)rm~(J5U(KZF$UeV)3GfLGsRMUjn!a7` zhZ?f~Mrm1_5(d6;uyNoYg+2cv(0D|!t7q6l#$d$Vr$H^hjIwD^nNu&`twFA%*!e6i z&;Hg6lEtUreW>e1g120?g|hFuMM;Bf_}EiKHhVMerNMIvrovZ%@k1@%o?&AZo@_7Q z8#OObT8GzhPee$oWN9I{I3*gst2g3n#^}KX@ zf=Cp;@j?Ov5$f^z-_Mq=eo-h8%xH@O9QeEriBJ$&pHc>r@6A5M*K4@F>BiT4p<2Ve z)*LS&l~2sUDX&lv&N$;e$Pbo;wGEb&huxOstL(e1fdJWR0>|EtsoRD*6QJJ6+{&1Z z+wle^Uo1QpC0M`V(fKJq@3>MIgq33F6bt~HB*0q$7ijlT@wc6`+p_iw@h1ax*+2O*Q?82sm;fdf($WB+sS1gWCPO zwYvvG8>n*%q*XP^*92-s1DH%u0!JOK76HEBLwK~xalNCH?Exan#k|%%y!uT>|B4JHsI3-3 zeean%2on&;{xpKKvs4E&=128X)LB71*(oJuqpr;_;)qg>_sMifi~xHpiQa)p#deak zOor`QD?cZ8o^u0-SX_A^Lh+}w0!Jrw&wXO^z=!~=3;cZk70=|VUH$42hJ-gH19cgw z+0jkd&MkG<*JXl?wyqDAXw@W>92DWN{**I zi*|5m=7;WU{1G}D|AXtxcb(*SDw2gGj<&ifo;H?j7iFLSPRgv@g>-4`d$)n~DF6jx zc*ILx7unHkVBBJ$(PvjRQ)t?+H8#hVpB{3$s>E`=#>qRb8j`S#8gm$s9rSU?Xr_}} zU+~%Xcze5|mVv{hhwxrk$bshOBhV8>YxL}~LtERpzS}X_1qediUL*A^By2YB+<_}S zyM-;bOQfRK*iAgY@O_oG+ndIr2tqzz0)J~zHk7W?xmd}wm^~3kufR6FY*7uE z?(o6($pKiDCetd(oS6%uxgV0BZg+zt?N4306{G}s=CUr(k2t|ZLl zqksF_tF)&$Qtnzu?Ty)?t4>QQ;)PjW_L-_T&#{TQLS0|(l~-SU>|7Qa#}hdDNcrWy z`(c6ygm~N>E?Pyxr^W!f77a@8RqM(-^sylu?`WH$0C;Kcygodx`GC~PG})Y>f5X9>hd@~Q^B7HXhkKbTwbvP$h1<9sj@0nt49ubjJzL6nC zZy0xy8|yss^S=@>jp+1p8c zfRY4=8}lH68=}oP9S}ze8@Sm_Z+l?k1KLN>eedSHO${XHfKzPcA6>o>GDYI+NAZ$` ze5%)~Yn`bSD~DM#K^n>9ktw-y;>p<#UQre}r8#-({L7W5fozK6&Idq`3O>0u;TdJa z=BzcgbZ+UvWK}D89j7V#F_h@0GHwmza#q@w5ZegpCual*_LYji5;m(3^9X)wp2EGx zus;d-wbwo?k$>c;$5sBB{q`gF&fWwGUjB(G@RgLaJ(j#1OV+5UJ+_7gEQ`+86ItMV zbtS=PZ0JwEV+p8gHvlQR+%!?&J?t@=9`c|3dCgSmN}Q%=$axj~3pEk8(aRIqB^BEO zpV+}(WsDWz4nD?N3>iIO7su|uaDF3fupqM%YY`%cjfg(j|lJ zbRq4sq0@%BT&>->WI))D8FS{C%>VFYaEtq9>9+Kdv5U@~e{<;!lR9h3`o*|G zt-uN9EFBrSs5V?O4jXaEW63yUl(8ei-38U0>ccz6_X7j61Xkai0^)u;dwQCM#JahC z63e8TC=X6FV##gl{FMGzTyq+XI^$NiTTNEBbuZsyarBdFTPw|!fgI@jEqkmQuRvds z$S8Ggf?Ba2e$oY5hgj2|8nwJ9fZ~-m7`3gjLxmHpCa(&50*&89z@~LudZ;UTD%_>f zLZHWp0$H=^0}_Q)i;F7KG+J7ezil4hDm6P#wv(dGLKR%#q&q*+EIYEavgs@x}0y{ zf%M%4uz7SD*29-=9&-^|=$w9}mHc+T4>C(di#0(ph>h@m8 z7t(7R!SER>eH?O&GBlNucJIEfwPFz*-N+96IFj1Y4+iwbVyP>=woeKuv5Q;dL(?#i zL9TQ(_%-n0_0~exuisfWC?)jDSk;v)mes0bE#1s)9ivLR%U5pnrZH~ODyGjfBu5s~ z+|BK$qC=c=h4D$ND#|d$4SX}njAcPOmcHHN#uom)+d=t=PN25dEAZ11=;A7g&U};e zv?MGw0Pd!VQKdLsDeV}bJED=C z7riU}CY>@SF5dBOsqdy`EC-;s$ic9G;7wk`F9bhawFJIDAkQk(!o-rwAkI;8SHq;e`?!J&qFPz6)pz^r(zYP4 zIWjDAb5BCIF}*qD#+4Nw1nvtj77pWP`)sXsRx;@>b(vqb+uIOk>jEQf*(Pqx`q~yJ z1_?Uo{@PcaLZ67_g(T1A)xue{=*aLf4<@Y%2zq{E!pvK>4A=XobpzNKR6gxUp`?jN z+o2}j9vB=7=~8HuStw>c4iXpOcGwJaFJDQ_HN)Fb(Wp1<73i@<#spsr?Ez*xvU&7H zAZ!G3FhBuL6Q>t;^-yUa7eti{bv8_D?g+qM;+yTAa|VOnk3DM}j)VEDV*-+o5xv=f zWmDe*_5$+{ub^IE;Jg4|ZR{pi*zq~vGY~T*Dz7L1h{8X8bm%DA+A3%}R`AFBTpYSKT09ajb>y)2It;?|4HMGuD8=~Qge-Ku-N zdsu(pUoye5wZCTsacE19Kr0x=-RCSWi>H4HmL+ozb$L*NV#BPo7mPVX__b*LVbkaM zU_}aPZy&cD!d-L1zz5D|vAURAbCnj|h0#4nl+v%E_Y-r00X!^}2?u%(#0Nak>{memIe@m<&NI z^|a-7)Nn`p38r--5cQixq=y50ES}+hOUcknI)AyK8}?_9^2~^v98AhDBfEo#NOKt6 za&JxiKr(qH_CWM|il`CZ{DKWN|480K6YuJ@bL^uHH2UX3vK|{e+HqdQcfX%&@^!DH zUzD1IlLz$xx~Wvo@tS_!=ojo?l+V^wP1LoKg$}sEPm;L4xI05F<^v*kCvOqk6(}H{ zJVnGRJP}N=2C9qzm4u66RCcgrN&Em{;yWIoQf#yRDNL?GUur{~da;6OU87>mMot0! zYY$~B^ei(U(c0<~Zk$YFfr%8x(9zX%1?(i5@E>P^S^g95y!*JbP$ zC$y1+vcEtv_Wdf!*usmIpR$sT*+teBy@q>%d17bCW>{kk#|A zqx7*p1U_`@6^J%Wh_Przc~YrshkD;+M^}AO*Vbza(!6kng0*btSH`m3muqyuAriW9 zp5~jlESRq^_HnYVB7Z-n`LZ2J0U5$?CejHQMi~!aKhF&{`jwE4#$2SRUSTTcLN;u zK>zC_QsOj9YUy%o*K)?r@S}5RO%*3hC-SSO%UsVd$RR`5{K)tmtqazl;W_{WuMh^E zWe}RMJhJ5695hyn|0{hGg6-nTm(%;nmaRL92RZ%7r5X*rjKT;hy^LO;+PcQF5{Xk+ zjOSBb1Gx~2Rrp9kdM=$_hyKe^9L1M$%F-BDMw($9bkQg=Jre1^oljZ2jqg4@`6%cUUdw z3%J_Gy_#~8x`Wp#1WuQ7tOS8yz)<))nczZ|dpZEob}!88#iV-X9mzF8QeZHDwS`-=RN?yJgg*PHgRlIy*gRKv&z$W z5*-{A(LfNMr}Vg^qkQ#(fj+~RsrX4c(Z-;^G;ZyXaIIu^r4Wv^^5DnQb)8Nz1LN!9HQt1>{eu9Yjw0vek)QsxTReur&%LM?1zCSnv~3q}Q}yVH6|sv0)H z!H`J61(DqO?Ibci>!b; z35y3y#F|)Sbq)Dm!=o5;RARX%{$aVFD7Bjn*uB-NSYu8zD<#WJty{*OaQYyj6U4i+ z)r7D28GwNniuc#6yRJ-zHPkaUSxJiqyStR}lzMTc8|-a4hybsdSeACg@WNj6DTT?Z z+?m|j=G|(TQD}gj&xX~K7+c71|K3S&cS1g}R{{m`%MU1+JZbZyQY6B9uTBsk%#j*i zZ6+;eh@hC@$FI_;xDGmRYn(*)boJ(6jgm=^iT${nWC4fm)}=H?9M>fOKu6o1G7-r( zshTJKo(2q4RRb-y+p*jLX2ibjQpk^sW{#EW697Hm0@v$Z3heGvTMUZlg;oGNER)7< z1&i~=x8hKD>Typ_rJmXNk75*j0f7Re2_?NV-9*$9T4fc;WXN7EbGzbl_ru$P2Z%Wz zjM=MQl`=%0ro5Bbi8Q0%w6A7Ert0KM@VPuK!$2ir` z5`RGxiC{Z9s!Uc@-IcVRg5XJ<;@I9Amu*W8x+urN@)FJmshR2lB@X)FqX4VSn6g)y z3!_@2jJ--8bgdb#pSoWiQcu&0of_BgieoDd;@w!9wwWdJdFm5aoOH=eQ$#(iA6I2m zLMcEYqaJP_0J}dW%uyV4BAoe|70Pe*8los2<=zBx@Zq*wnq7)%nzzy?nT<_Lyi(k_!byxqr?k)*2X+lu$O}s$E=k98imH!84>&M-7e??S>&gp~O3F#dj9> zMARIb%lL*`)@X4CuwN>gbABDq5YQJK1dm~!fi>-YIm@xh?hlQjVL$`f@QQv(!I43R zQyoi6fIuiA-k@?BWTDSYp|!kV3wP#YO|Cs{%>+nu7oM(M&Bj2pHh>a?AX?wp@YI;MZ1G!7wDT>EFgGz!%Yc zUta=1n~i>z@ zd_n=@ByhraLo_KMX+Q@JgyeU#FpDgv32qk{EMxf~@VBMcMBrm*1lS>4NV{hAt|gVw zW{8ahVtYk{gmVkGqs6`wZ331z&Kp|JN|+|$@RqhuX-Y)|EQ0)zbWP9zaff{l;%A@< zmz^K|1HA9u3f!8|4RreK+dp*R`y)`(l&D!$G@Ac9rPrmm_#bEQ!OV{;F1HxT@|FP0?nQE^|bRr6=B1GX?UJcUk?cgh|{3Mv_hm5d<+1| z<8XK05i+L+Q|aTm=-sn{aMA*XC?pb#>2ucsFsu7&aTtm+oFYJW@+}aS76E_TYt(Wf zMW%+iDaz`Yx)McOIe7S*YOfRX|42yg=^k@;QOVk+!P&FnR6OQ61tdJu@m*4pmwyap zBJ{-7gR^k@NZ5SlpQ-nf$1g>DlF&Tgks;%n0=J8RN{t5>DowmZWPB>x$6bdFt4d+U3ugN zC;+?Ie@M>V)YD6w01FRM2`ct2$8Ou&Iz|PWx2cZqrZ>K%DS@Jbw{L!ML7(E+VJy^o zw?PmJR@ezDA`GJZpV0;sDS1bA^vh{8qs&jsq#bX1(eQl3$)1tR&97zh#67Om-Fo(t z5?~(@?%S*jhG^1>+KCqx%P|3$j4fvEPDiXDE@bwPWuO|WO6}VkUD>8nk^>plS(~?6LVnsGtlybj-qr1Wu~>7D>=N>qwInvm`LOQP18l@OZ`kjeR_rkQstv` zVNMBT180z+#A(axhB<;{$=kxbg%T3iVVuAxtjYSR$)@Un^CvmUzln`Cgl#Sw3}#hJ zEKdHRbK-`^ag~+<9lRXAl8jh^mUsX{8;X&42!2uQvOV?cHkuDlhrZ}>%s-QDkoIp? z>=BHkk`+9>)EPfO0S(Uk;=~Up3@0Zt(|zV@2=X_0-qjjvx?eBkj$=lEcSO(g=jc?; zS?i&zo{z{2A?XhfyAI5it6m4GEJ=0pAj>!fGh!w`#ekz%z446$r3i{@<$F3T$+tXI zWSIHudDN4^m`g0N-h7IB5iCd#rt;HAzNHENsWe?65YTve{dLb)Qdb`b^>YsSw%prr z({Af_4DNTGa8FodT%9NBy>VjucD5$@n`xo}$=agMHWyEzO*ct9*A@M0lue6%j9~$- zo0EhWvrB>Op&F#|H-lsCla(PKn$pR`P`}p4O@^I7c1I%b(crW^7RSd4?)A+!r`zrh ztUp`nSTIDC;rss%g?&~WTCXfm4c>!?V3UGQ3|%u1Q)^b)6@R9T*3})zBrSoUG~N2? z9Q?>e01>}}3GSH_Wd*IUQVxbRV_8x~3T9r?qJguK9+KfYJf| z6zHK`Jj$|9Ef_NRn>G2@VC1s`yn%e$(UDznp1XN3$eS0&f|-oDmbiw7&*kMLd!RTN zd~=-a0V=p_4VwZUpRRq_>T}Mt>7X1nJfh;fHY<#3*#1bO;8!h6A#?iX5L_q1;+=kyCj^@6OfclL77k%XJ7mb#muIaJ_oi0QqQ_SCvX_WM;! zP5HQB2-Kt)j&M$!en~eCno~`w1@ZeM-~O8)aG`ReW6c7uKunJW&D2+<(|msOTv=X1 zU}O{0$&<4zwG%K3DAejr?Q4E!P-ieR*m*rTDd{)BKG|$|kLmX`A%~534Kx}hYC-Q? zl%0?`iYhh)(P4S_wEy#%IJ%^qnlvYCTeXRWPs8JNUWMH|wkeZRyw1tR@A=X@Z6VvH z$Ivu(E4vz-zxY7iT52Y&VN;-}sLu75VT6f05-h8E-r{(dVC+*B%us^~`@{5z<>B}q_#el^_#O(zEZu|cX z7xdu0{Y6T&j!8FJMv_B-L$?2COms3+?%ag!>YvdOjmntBfGAH zwF$F0#QY+H;@{v}0~^9nvm>%x)Ud6RWD#uW0*)cl8)33kj)xXvcxYk?{4FYAahK<9 zF}%c^Jb-K-Rj(tw^!m%XdP!@2D!jkzhcfiG{jNTJkAIiaZv@n*_^HT)^q;#`*~DH> zQyoTF1Rng(ApKlop^Y?@HI;IthOin!Tl;%bI1||ipW~5!{=Kv$7I!I@=)Tu13S_#L zczDM?KN_PT;_%*7nbIhd{Sv?8%Ze?5bkbJ42Mi?%)4!uXHlk~*H5d^bL*KbnO&uc6%1(`D-6331Gjp!rPoE)bz6BB4s zw#-!@#?fv)$aYviuJn5Q`}O3^>)U)(J!Zk2xH$vX$pu#Z`o`8D`$0uf{Y)`7{NG^a zAe^X?#^uaSq?*M+kjnf}k=%V#f0&N;`PJq7!eU=oTHf@jxEGdm*7agx^-G#Lu@?dg zA36o6Z4RwR-(nU`Yn^KaG8azhe^sQP5O-Vl|>1y2ZtE3805so5{ z&~Rqigu90>SjQUpRPG~t`tRTL3Uk_eNq9$nx?pu&?6XpAMd4Fkb0EyXlI~vo*Qm-=tBTQ9C}y8xZ)Z%{9);o;&$4VGDV6 zS>t>Oj=0&ASp=kXSH-Cx|346;8RDe}`(ke7V=tx09Xh{b8R@E7z2(X=f&UF6;EECebw6Dq)2-*55hkU7V?$Y42B}NmdQIP7Jd0FN z)4OtGT*x8u%|Sr=s9>{iBKtU~YT>O8`8RcJNJMZJn_fu=(kOy3FUg8)=$;oA-~hRs zg7}xkcX8UB8A9O*g>@_^cs`FH!;y)bZbmexL(v=DA^V7m1y@R2O;FAieG%F{dLR8V zxeR=;wrROc_VS>}7Qqmqkbv;@T+WjP_@Q6L!&;=MH#%@~X?9_BWoa>GWjH!@a%?SZ zX>)WoG&*Q>bZ~PzFE40yXJ=_{XJ%nzWiDfHFKujiEn{zOZFO#Gba^jPcV%ufEi*7e WEkQFvEi__jbT4vcb8l>RWiMrSovs-G literal 0 HcmV?d00001 diff --git a/rust/exo_pyo3_bindings/Cargo.toml b/rust/exo_pyo3_bindings/Cargo.toml index 12803ab4..5cfd5e5a 100644 --- a/rust/exo_pyo3_bindings/Cargo.toml +++ b/rust/exo_pyo3_bindings/Cargo.toml @@ -23,6 +23,7 @@ workspace = true [dependencies] networking = { workspace = true } +downloads = { workspace = true } # interop pyo3 = { version = "0.27.1", features = [ diff --git a/rust/exo_pyo3_bindings/src/downloads.rs b/rust/exo_pyo3_bindings/src/downloads.rs new file mode 100644 index 00000000..4e1e9663 --- /dev/null +++ b/rust/exo_pyo3_bindings/src/downloads.rs @@ -0,0 +1,327 @@ +//! Downloads module - BitTorrent downloads PyO3 bindings + +use crate::ext::*; +use downloads::bencode::AnnounceParams; +use downloads::tracker::{PeerInfo, TopologyData, handle_announce as rust_handle_announce}; +use downloads::{DownloadProgress, TorrentSession}; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyDict}; +use std::net::Ipv4Addr; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::Mutex; + +/// Handle a tracker announce request +/// +/// Args: +/// params: Dictionary with announce parameters (info_hash, peer_id, port, etc.) +/// peers: List of peer dictionaries (node_id, ip, port, has_complete, priority) +/// +/// Returns: +/// Bencoded announce response as bytes +#[pyfunction] +fn handle_tracker_announce( + py: Python<'_>, + params: &Bound<'_, PyDict>, + peers: &Bound<'_, pyo3::types::PyList>, +) -> PyResult> { + // Parse announce params + let info_hash = { + let info_hash_item = params + .get_item("info_hash")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing info_hash"))?; + let info_hash_bytes: &[u8] = info_hash_item.extract()?; + + if info_hash_bytes.len() != 20 { + return Err(pyo3::exceptions::PyValueError::new_err( + "info_hash must be 20 bytes", + )); + } + + let mut info_hash = [0u8; 20]; + info_hash.copy_from_slice(info_hash_bytes); + info_hash + }; + + let peer_id = { + let peer_id_item = params + .get_item("peer_id")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing peer_id"))?; + let peer_id_bytes: &[u8] = peer_id_item.extract()?; + + if peer_id_bytes.len() != 20 { + return Err(pyo3::exceptions::PyValueError::new_err( + "peer_id must be 20 bytes", + )); + } + + let mut peer_id = [0u8; 20]; + peer_id.copy_from_slice(peer_id_bytes); + peer_id + }; + + let port: u16 = params + .get_item("port")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing port"))? + .extract()?; + + let uploaded: u64 = params + .get_item("uploaded")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing uploaded"))? + .extract()?; + + let downloaded: u64 = params + .get_item("downloaded")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing downloaded"))? + .extract()?; + + let left: u64 = params + .get_item("left")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing left"))? + .extract()?; + + let compact: bool = params + .get_item("compact")? + .map(|v| v.extract().unwrap_or(true)) + .unwrap_or(true); + + let announce_params = AnnounceParams { + info_hash, + peer_id, + port, + uploaded, + downloaded, + left, + compact, + event: None, // TODO: parse event if needed + }; + + // Parse peer list + let peer_infos: Result, PyErr> = peers + .iter() + .map(|peer_item| { + let peer_dict: &Bound<'_, PyDict> = peer_item.downcast()?; + let node_id: String = peer_dict + .get_item("node_id")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing node_id"))? + .extract()?; + + let ip_str: String = peer_dict + .get_item("ip")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing ip"))? + .extract()?; + + let ip: Ipv4Addr = ip_str + .parse() + .map_err(|_| pyo3::exceptions::PyValueError::new_err("Invalid IP address"))?; + + let port: u16 = peer_dict + .get_item("port")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing port"))? + .extract()?; + + let has_complete: bool = peer_dict + .get_item("has_complete")? + .map(|v: Bound<'_, pyo3::PyAny>| v.extract().unwrap_or(false)) + .unwrap_or(false); + + let priority: i32 = peer_dict + .get_item("priority")? + .map(|v: Bound<'_, pyo3::PyAny>| v.extract().unwrap_or(0)) + .unwrap_or(0); + + Ok(PeerInfo { + node_id, + ip, + port, + has_complete, + priority, + }) + }) + .collect(); + + let peer_infos = peer_infos?; + + let topology = TopologyData { peers: peer_infos }; + + // Call Rust tracker handler + let response_bytes = rust_handle_announce(&announce_params, &topology).pyerr()?; + + // Return as Python bytes + Ok(PyBytes::new(py, &response_bytes).unbind()) +} + +/// Get an embedded torrent file +/// +/// Args: +/// model_id: Model identifier (e.g., "mlx-community/Qwen3-30B-A3B-4bit") +/// revision: Git commit hash +/// +/// Returns: +/// Torrent file contents as bytes, or None if not found +#[pyfunction] +fn get_embedded_torrent( + py: Python<'_>, + model_id: String, + revision: String, +) -> PyResult>> { + match downloads::get_embedded_torrent(&model_id, &revision) { + Some(data) => Ok(Some(PyBytes::new(py, &data).unbind())), + None => Ok(None), + } +} + +/// Python wrapper for TorrentSession +#[pyclass] +struct TorrentSessionHandle { + session: Arc>, +} + +#[pymethods] +impl TorrentSessionHandle { + /// Create a new torrent session + /// + /// Args: + /// session_dir: Directory to store session state and downloads + #[new] + fn new(session_dir: String) -> PyResult { + let session_path = PathBuf::from(session_dir); + + let session = tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { TorrentSession::new(session_path).await }) + .pyerr()?; + + Ok(Self { + session: Arc::new(Mutex::new(session)), + }) + } + + /// Add a torrent from bytes + /// + /// Args: + /// torrent_data: Raw .torrent file contents + /// save_path: Where to save downloaded files + /// file_indices: Optional list of file indices to download + /// + /// Returns: + /// Info hash as hex string + fn add_torrent( + &self, + py: Python<'_>, + torrent_data: Vec, + save_path: String, + file_indices: Option>, + ) -> PyResult { + let session = Arc::clone(&self.session); + let save_path = PathBuf::from(save_path); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { + session + .lock() + .await + .add_torrent(torrent_data, save_path, file_indices) + .await + }) + .pyerr() + }) + } + + /// Get download progress for a torrent + /// + /// Args: + /// info_hash: Torrent info hash + /// + /// Returns: + /// Dictionary with progress information + fn get_progress(&self, py: Python<'_>, info_hash: String) -> PyResult> { + let session = Arc::clone(&self.session); + + let progress: DownloadProgress = py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { session.lock().await.get_progress(&info_hash).await }) + .pyerr() + })?; + + let dict = PyDict::new(py); + dict.set_item("downloaded_bytes", progress.downloaded_bytes)?; + dict.set_item("total_bytes", progress.total_bytes)?; + dict.set_item("download_speed", progress.download_speed)?; + dict.set_item("upload_speed", progress.upload_speed)?; + dict.set_item("peers_connected", progress.peers_connected)?; + dict.set_item("is_finished", progress.is_finished)?; + + Ok(dict.unbind()) + } + + /// Wait until torrent download is completed + /// + /// Args: + /// info_hash: Torrent info hash + fn wait_until_completed(&self, py: Python<'_>, info_hash: String) -> PyResult<()> { + let session = Arc::clone(&self.session); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { session.lock().await.wait_until_completed(&info_hash).await }) + .pyerr() + }) + } + + /// Enable seeding for a torrent + /// + /// Args: + /// info_hash: Torrent info hash + fn enable_seeding(&self, py: Python<'_>, info_hash: String) -> PyResult<()> { + let session = Arc::clone(&self.session); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { session.lock().await.enable_seeding(&info_hash).await }) + .pyerr() + }) + } + + /// Remove a torrent from the session + /// + /// Args: + /// info_hash: Torrent info hash + fn remove_torrent(&self, py: Python<'_>, info_hash: String) -> PyResult<()> { + let session = Arc::clone(&self.session); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { session.lock().await.remove_torrent(&info_hash).await }) + .pyerr() + }) + } + + /// List all torrents in the session + /// + /// Returns: + /// List of info hashes + fn list_torrents(&self, py: Python<'_>) -> PyResult> { + let session = Arc::clone(&self.session); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { Ok(session.lock().await.list_torrents().await) }) + }) + } +} + +/// Downloads submodule +pub(crate) fn downloads_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(handle_tracker_announce, m)?)?; + m.add_function(wrap_pyfunction!(get_embedded_torrent, m)?)?; + m.add_class::()?; + Ok(()) +} diff --git a/rust/exo_pyo3_bindings/src/lib.rs b/rust/exo_pyo3_bindings/src/lib.rs index 4f591b8c..6a78971f 100644 --- a/rust/exo_pyo3_bindings/src/lib.rs +++ b/rust/exo_pyo3_bindings/src/lib.rs @@ -17,10 +17,12 @@ extern crate core; mod allow_threading; +pub(crate) mod downloads; mod examples; pub(crate) mod networking; pub(crate) mod pylibp2p; +use crate::downloads::downloads_submodule; use crate::networking::networking_submodule; use crate::pylibp2p::ident::ident_submodule; use crate::pylibp2p::multiaddr::multiaddr_submodule; @@ -207,6 +209,7 @@ fn main_module(m: &Bound<'_, PyModule>) -> PyResult<()> { ident_submodule(m)?; multiaddr_submodule(m)?; networking_submodule(m)?; + downloads_submodule(m)?; // top-level constructs // TODO: ... diff --git a/scripts/mktorrent.sh b/scripts/mktorrent.sh new file mode 100755 index 00000000..a6debcbc --- /dev/null +++ b/scripts/mktorrent.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p mktorrent -p python3Packages.huggingface-hub -p git -p git-lfs +set -euo pipefail +set -x + +MODEL="$1" + +mkdir -p "$MODEL" + +# Step 1: Clone/fetch the repo and get the hash of head +mkdir -p "$MODEL" +if test -d "$MODEL/git"; then + # Assert that the origin is correct + git -C "$MODEL/git" fetch +else + git clone "https://huggingface.co/$MODEL" "$MODEL/git" +fi + +HASH=$(git -C "$MODEL/git" rev-parse origin/main) +LARGE_FILES=$(git -C "$MODEL/git" lfs ls-files --all --name-only) + +SMALL_DIR="$MODEL/$HASH-small" +LARGE_DIR="$MODEL/$HASH-large" +mkdir -p "$SMALL_DIR" "$LARGE_DIR" + +# Step 2: Prepare files. Two torrents: one for large files and one for metadata. +git -C "$MODEL/git" archive "$HASH" | tar -x -C "$SMALL_DIR" +echo "$LARGE_FILES" | xargs -I{} rm "$SMALL_DIR/{}" + +echo "$LARGE_FILES" | xargs hf download "$MODEL" --revision "$HASH" --local-dir "$LARGE_DIR" --cache-dir "$(realpath .cache)" --include +if test -d "$LARGE_DIR/.cache"; then + echo ".cache created against our wishes, deleting it..." + rm -r "$LARGE_DIR/.cache" +fi + +# Step 3: Create both torrents +mkdir -p "torrents/$MODEL/" +SMALL_TORRENT_PATH="torrents/$MODEL/${HASH}.small.torrent" +LARGE_TORRENT_PATH="torrents/$MODEL/${HASH}.large.torrent" + +mktorrent "$SMALL_DIR/" --output="$SMALL_TORRENT_PATH" \ + -n "$HASH" \ + --web-seed="https://huggingface.co/$MODEL/raw/" \ + --no-date \ + --announce="udp://tracker.opentrackr.org:1337/announce" + # --private + +mktorrent "$LARGE_DIR/" --output="$LARGE_TORRENT_PATH" \ + -n "$HASH" \ + --web-seed="https://huggingface.co/$MODEL/resolve/" \ + --piece-length=24 \ + --no-date \ + --announce="udp://tracker.opentrackr.org:1337/announce" + # --private + +echo "Successfully created torrent files in:" +echo "$SMALL_TORRENT_PATH" +echo "$LARGE_TORRENT_PATH" diff --git a/src/exo/main.py b/src/exo/main.py index 43d69aa5..a8714d5b 100644 --- a/src/exo/main.py +++ b/src/exo/main.py @@ -35,6 +35,7 @@ class Node: api: API | None node_id: NodeId + enable_torrents: bool _tg: TaskGroup = field(init=False, default_factory=anyio.create_task_group) @classmethod @@ -66,7 +67,8 @@ class Node: worker = Worker( node_id, session_id, - exo_shard_downloader(), + exo_shard_downloader(enable_torrents=args.enable_torrents), + initial_connection_messages=[], connection_message_receiver=router.receiver(topics.CONNECTION_MESSAGES), global_event_receiver=router.receiver(topics.GLOBAL_EVENTS), local_event_sender=router.sender(topics.LOCAL_EVENTS), @@ -74,7 +76,6 @@ class Node: ) else: worker = None - # We start every node with a master master = Master( node_id, @@ -98,7 +99,7 @@ class Node: election_result_sender=er_send, ) - return cls(router, worker, election, er_recv, master, api, node_id) + return cls(router, worker, election, er_recv, master, api, node_id, args.enable_torrents) async def run(self): async with self._tg as tg: @@ -175,7 +176,7 @@ class Node: self.worker = Worker( self.node_id, result.session_id, - exo_shard_downloader(), + exo_shard_downloader(enable_torrents=self.enable_torrents), connection_message_receiver=self.router.receiver( topics.CONNECTION_MESSAGES ), @@ -215,6 +216,7 @@ class Args(CamelCaseModel): api_port: PositiveInt = 52415 tb_only: bool = False no_worker: bool = False + enable_torrents: bool = False @classmethod def parse(cls) -> Self: @@ -256,6 +258,12 @@ class Args(CamelCaseModel): "--no-worker", action="store_true", ) + parser.add_argument( + "--enable-torrents", + action="store_true", + dest="enable_torrents", + help="Enable BitTorrent-based downloads (experimental)", + ) args = parser.parse_args() return cls(**vars(args)) # pyright: ignore[reportAny] - We are intentionally validating here, we can't do it statically diff --git a/src/exo/master/api.py b/src/exo/master/api.py index 30f87e2a..79b6720c 100644 --- a/src/exo/master/api.py +++ b/src/exo/master/api.py @@ -5,9 +5,9 @@ from typing import cast import anyio from anyio import create_task_group from anyio.abc import TaskGroup -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse +from fastapi.responses import Response, StreamingResponse from fastapi.staticfiles import StaticFiles from hypercorn.asyncio import serve # pyright: ignore[reportUnknownVariableType] from hypercorn.config import Config @@ -178,6 +178,7 @@ class API: self.app.post("/bench/chat/completions")(self.bench_chat_completions) self.app.get("/state")(lambda: self.state) self.app.get("/events")(lambda: self._event_log) + self.app.get("/_internal/announce")(self.tracker_announce) async def place_instance(self, payload: PlaceInstanceParams): command = PlaceInstance( @@ -622,6 +623,65 @@ class API: ] ) + async def tracker_announce(self, request: Request) -> Response: + """BitTorrent tracker announce endpoint for private tracker.""" + try: + from exo_pyo3_bindings import handle_tracker_announce # type: ignore + except ImportError as e: + raise HTTPException( + status_code=501, + detail="Torrent support not available (exo_pyo3_bindings not installed)", + ) from e + + # Parse announce parameters from query string + query_params = dict(request.query_params) + + # Extract required parameters + try: + info_hash_hex = query_params.get("info_hash", "") + peer_id_hex = query_params.get("peer_id", "") + + # URL decode and convert to bytes + info_hash = bytes.fromhex(info_hash_hex) if info_hash_hex else b"" + peer_id = bytes.fromhex(peer_id_hex) if peer_id_hex else b"" + + if len(info_hash) != 20 or len(peer_id) != 20: + raise ValueError("info_hash and peer_id must be 20 bytes") + + params = { + "info_hash": info_hash, + "peer_id": peer_id, + "port": int(query_params.get("port", "6881")), + "uploaded": int(query_params.get("uploaded", "0")), + "downloaded": int(query_params.get("downloaded", "0")), + "left": int(query_params.get("left", "0")), + "compact": query_params.get("compact", "1") == "1", + } + except (ValueError, KeyError) as e: + raise HTTPException( + status_code=400, + detail=f"Invalid announce parameters: {e}", + ) from e + + # Build peer list from topology + # TODO: Implement _build_peer_list_from_topology() to extract peers from self.state.topology + peers = [] # For now, return empty peer list + + # Call Rust tracker handler + try: + response_bytes: bytes = handle_tracker_announce(params, peers) # type: ignore + return Response( + content=response_bytes, + media_type="text/plain", + headers={"Content-Type": "text/plain"}, + ) + except Exception as e: + logger.error(f"Tracker announce error: {e}") + raise HTTPException( + status_code=500, + detail=f"Tracker announce failed: {e}", + ) from e + async def run(self): cfg = Config() cfg.bind = f"0.0.0.0:{self.port}" diff --git a/src/exo/shared/types/models.py b/src/exo/shared/types/models.py index e99a7667..a6bf6c62 100644 --- a/src/exo/shared/types/models.py +++ b/src/exo/shared/types/models.py @@ -16,3 +16,4 @@ class ModelMetadata(CamelCaseModel): n_layers: PositiveInt hidden_size: PositiveInt supports_tensor: bool + revision: str | None = None # Git commit hash for torrent lookup diff --git a/src/exo/worker/download/impl_shard_downloader.py b/src/exo/worker/download/impl_shard_downloader.py index 46f55ff9..f045c396 100644 --- a/src/exo/worker/download/impl_shard_downloader.py +++ b/src/exo/worker/download/impl_shard_downloader.py @@ -12,10 +12,17 @@ from exo.worker.download.download_utils import RepoDownloadProgress, download_sh from exo.worker.download.shard_downloader import ShardDownloader -def exo_shard_downloader(max_parallel_downloads: int = 8) -> ShardDownloader: - return SingletonShardDownloader( - CachedShardDownloader(ResumableShardDownloader(max_parallel_downloads)) - ) +def exo_shard_downloader( + max_parallel_downloads: int = 8, enable_torrents: bool = False +) -> ShardDownloader: + if enable_torrents: + from exo.worker.download.torrent_downloader import TorrentShardDownloader + + base = TorrentShardDownloader(max_parallel_downloads) + else: + base = ResumableShardDownloader(max_parallel_downloads) + + return SingletonShardDownloader(CachedShardDownloader(base)) async def build_base_shard(model_id: str) -> ShardMetadata: diff --git a/src/exo/worker/download/torrent_downloader.py b/src/exo/worker/download/torrent_downloader.py new file mode 100644 index 00000000..ea9c0cc0 --- /dev/null +++ b/src/exo/worker/download/torrent_downloader.py @@ -0,0 +1,212 @@ +"""Torrent-based shard downloader using BitTorrent protocol with private tracker. + +This module implements downloading model shards via BitTorrent with: +- Private tracker integration (/_internal/announce endpoint) +- WebSeed (BEP 19) fallback to HTTPS +- Selective file download based on shard layer ranges +- Persistent seeding of downloaded content +""" + +import asyncio +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Callable + +from loguru import logger + +from exo.shared.constants import EXO_MODELS_DIR +from exo.shared.types.worker.shards import ShardMetadata +from exo.worker.download.download_utils import RepoDownloadProgress +from exo.worker.download.shard_downloader import ShardDownloader + + +class TorrentShardDownloader(ShardDownloader): + """Download model shards using BitTorrent with private tracker.""" + + def __init__(self, max_parallel_downloads: int = 8): + self.max_parallel_downloads = max_parallel_downloads + self._progress_callbacks: list[ + Callable[[ShardMetadata, RepoDownloadProgress], None] + ] = [] + + # Initialize TorrentSessionHandle + try: + from exo_pyo3_bindings import TorrentSessionHandle # type: ignore + + session_dir = EXO_MODELS_DIR / "v2" / ".torrent_session" + session_dir.mkdir(parents=True, exist_ok=True) + self.session = TorrentSessionHandle(str(session_dir)) + except ImportError: + logger.error("exo_pyo3_bindings not available, torrent downloads disabled") + self.session = None + + def on_progress( + self, callback: Callable[[ShardMetadata, RepoDownloadProgress], None] + ) -> None: + """Register a progress callback.""" + self._progress_callbacks.append(callback) + + async def ensure_shard( + self, shard: ShardMetadata, config_only: bool = False + ) -> Path: + """Download a model shard using BitTorrent. + + Args: + shard: Shard metadata including model ID and layer range + config_only: If True, only download config files (not implemented for torrents yet) + + Returns: + Path to the downloaded shard directory + + Raises: + RuntimeError: If torrent file is not found or session not initialized + """ + if self.session is None: + raise RuntimeError( + "TorrentSessionHandle not initialized. " + "exo_pyo3_bindings module not available." + ) + + model_id = str(shard.model_meta.model_id) + + # Resolve "main" branch to commit hash + revision = await self._resolve_revision(model_id, "main") + + # Load embedded torrent file + torrent_data = self._load_embedded_torrent(model_id, revision) + if torrent_data is None: + raise RuntimeError( + f"Torrent not found for {model_id}@{revision}. " + f"Expected at: rust/downloads/torrents/{model_id}/{revision}.torrent. " + f"Please add the torrent file or use HTTP downloads (--enable-torrents=False)." + ) + + # Build v2 path: ~/.exo/models/v2/{model_id}/{revision} + save_path = EXO_MODELS_DIR / "v2" / model_id / revision + save_path.mkdir(parents=True, exist_ok=True) + + # Calculate which files to download based on shard layers + # For now, download all files (selective download will be implemented later) + file_indices = None # None means download all files + + # Add torrent and download + logger.info( + f"Adding torrent for {model_id}@{revision} to session, saving to {save_path}" + ) + info_hash = self.session.add_torrent(torrent_data, str(save_path), file_indices) + + # Wait for download with progress reporting + logger.info(f"Starting download for {info_hash}") + while True: + progress_dict = self.session.get_progress(info_hash) + + # Convert dict to RepoDownloadProgress for callbacks + progress = RepoDownloadProgress( + downloaded=progress_dict["downloaded_bytes"], + total=progress_dict["total_bytes"], + ) + self._report_progress(shard, progress) + + if progress_dict["is_finished"]: + logger.info(f"Download completed for {info_hash}") + break + + await asyncio.sleep(0.5) + + # Enable persistent seeding + logger.info(f"Enabling seeding for {info_hash}") + self.session.enable_seeding(info_hash) + + return save_path + + async def get_shard_download_status( + self, + ) -> AsyncIterator[tuple[Path, RepoDownloadProgress]]: + """Get download status for all shards.""" + if self.session is None: + return + + # List all torrents in the session + info_hashes = self.session.list_torrents() + + for info_hash in info_hashes: + try: + progress_dict = self.session.get_progress(info_hash) + progress = RepoDownloadProgress( + downloaded=progress_dict["downloaded_bytes"], + total=progress_dict["total_bytes"], + ) + + # We don't have a straightforward way to map info_hash back to path + # This would require tracking in the session or metadata + # For now, yield empty path + yield (Path(), progress) + except Exception as e: + logger.error(f"Error getting status for {info_hash}: {e}") + + async def get_shard_download_status_for_shard( + self, shard: ShardMetadata + ) -> RepoDownloadProgress: + """Get download status for a specific shard.""" + if self.session is None: + return RepoDownloadProgress(downloaded=0, total=0) + + model_id = str(shard.model_meta.model_id) + revision = await self._resolve_revision(model_id, "main") + + # We would need to track info_hash -> shard mapping + # For now, return empty progress + return RepoDownloadProgress(downloaded=0, total=0) + + async def _resolve_revision(self, model_id: str, branch: str = "main") -> str: + """Resolve branch name to commit hash using HuggingFace API. + + Args: + model_id: Model identifier (e.g., "mlx-community/Qwen3-30B-A3B-4bit") + branch: Branch name (default: "main") + + Returns: + Git commit hash (SHA) + """ + try: + from huggingface_hub import model_info + + info = model_info(model_id, revision=branch) + return str(info.sha) + except Exception as e: + logger.error(f"Failed to resolve revision for {model_id}@{branch}: {e}") + # Fallback to "main" as revision if API call fails + logger.warning(f"Using branch name '{branch}' as revision") + return branch + + def _load_embedded_torrent(self, model_id: str, revision: str) -> bytes | None: + """Load embedded torrent file from Rust binary. + + Args: + model_id: Model identifier + revision: Git commit hash + + Returns: + Torrent file contents, or None if not found + """ + try: + from exo_pyo3_bindings import get_embedded_torrent # type: ignore + + result: bytes | None = get_embedded_torrent(model_id, revision) # type: ignore + return result + except ImportError: + logger.warning("exo_pyo3_bindings not available, cannot load torrents") + return None + except Exception as e: + logger.error(f"Error loading torrent for {model_id}@{revision}: {e}") + return None + + def _report_progress( + self, shard: ShardMetadata, progress: RepoDownloadProgress + ) -> None: + """Report progress to all registered callbacks.""" + for callback in self._progress_callbacks: + try: + callback(shard, progress) + except Exception as e: + logger.error(f"Error in progress callback: {e}")