diff --git a/Cargo.lock b/Cargo.lock index c54f01d1..20bd0d0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -9,7 +15,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ "crypto-common", - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -192,6 +198,15 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "155a5a185e42c6b77ac7b88a15143d930a9e9727a5b7b77eed417404ab15c247" +[[package]] +name = "assert_cfg" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e2651f366b7ee3f97729fded1441539b49d5f39eeb05b842689e11e84501b2" +dependencies = [ + "const_panic", +] + [[package]] name = "async-channel" version = "2.5.0" @@ -204,6 +219,19 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98ec5f6c2f8bc326c994cb9e241cc257ddaba9afa8555a43cffbb5dd86efaa37" +dependencies = [ + "compression-codecs", + "compression-core", + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-io" version = "2.6.0" @@ -222,6 +250,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "async-trait" version = "0.1.89" @@ -270,6 +320,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "backoff" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" +dependencies = [ + "getrandom 0.2.16", + "instant", + "rand 0.8.5", +] + [[package]] name = "base-x" version = "0.2.11" @@ -323,6 +384,35 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bincode" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" +dependencies = [ + "bincode_derive", + "serde", + "unty", +] + +[[package]] +name = "bincode_derive" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -335,6 +425,18 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake2" version = "0.10.6" @@ -350,7 +452,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array", + "generic-array 0.14.7", ] [[package]] @@ -387,6 +489,17 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + [[package]] name = "bumpalo" version = "3.19.0" @@ -472,6 +585,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link", ] @@ -520,6 +634,41 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "commoncrypto" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d056a8586ba25a1e4d61cb090900e495952c7886786fc55f909ab2f819b69007" +dependencies = [ + "commoncrypto-sys", +] + +[[package]] +name = "commoncrypto-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fed34f46747aa73dfaa578069fd8279d2818ade2b55f38f22a9401c7f4083e2" +dependencies = [ + "libc", +] + +[[package]] +name = "compression-codecs" +version = "0.4.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f7ac3e5b97fdce45e8922fb05cae2c37f7bbd63d30dd94821dacfd8f3f2bf2" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -541,6 +690,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f421161cb492475f1661ddc9815a745a1c894592070661180fdec3d4872e9c3" +[[package]] +name = "const_panic" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e262cdaac42494e3ae34c43969f9cdeb7da178bdb4b66fa6a1ea2edb4c8ae652" +dependencies = [ + "typewit", +] + [[package]] name = "convert_case" version = "0.10.0" @@ -584,6 +742,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "critical-section" version = "1.2.0" @@ -636,7 +803,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "subtle", "zeroize", @@ -648,11 +815,23 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ - "generic-array", + "generic-array 0.14.7", "rand_core 0.6.4", "typenum", ] +[[package]] +name = "crypto-hash" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a77162240fd97248d19a564a565eb563a3f592b386e4136fb300909e67dddca" +dependencies = [ + "commoncrypto", + "hex 0.3.2", + "openssl", + "winapi", +] + [[package]] name = "ctr" version = "0.9.2" @@ -735,6 +914,21 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", + "serde", +] + [[package]] name = "data-encoding" version = "2.9.0" @@ -804,6 +998,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -841,6 +1036,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "directories" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f5094c54661b38d03bd7e50df373292118db60b585c08a411c6d840017fe7d" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -852,12 +1068,36 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "downloads" +version = "0.0.1" +dependencies = [ + "anyhow", + "derive_more", + "futures", + "futures-util", + "include_dir", + "itertools 0.14.0", + "librqbit", + "log", + "serde", + "thiserror 2.0.17", + "tokio", + "util", +] + [[package]] name = "dtoa" version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6add3b8cff394282be81f3fc1a0605db594ed69890078ca6e2cab1c408bcf04" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "ecdsa" version = "0.16.9" @@ -912,7 +1152,7 @@ dependencies = [ "crypto-bigint", "digest", "ff", - "generic-array", + "generic-array 0.14.7", "group", "pem-rfc7468", "pkcs8", @@ -1000,6 +1240,7 @@ version = "0.0.1" dependencies = [ "delegate", "derive_more", + "downloads", "env_logger", "extend", "futures", @@ -1030,6 +1271,12 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "ff" version = "0.13.1" @@ -1052,6 +1299,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +[[package]] +name = "flate2" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1064,6 +1321,27 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1073,6 +1351,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.31" @@ -1204,6 +1488,15 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1284,6 +1577,29 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "governor" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9efcab3c1958580ff1f25a2a41be1668f7603d849bb63af523b208a3cc1223b8" +dependencies = [ + "cfg-if", + "dashmap", + "futures-sink", + "futures-timer", + "futures-util", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "nonzero_ext", + "parking_lot", + "portable-atomic", + "quanta", + "rand 0.9.2", + "smallvec", + "spinning_top", + "web-time", +] + [[package]] name = "group" version = "0.13.0" @@ -1307,13 +1623,19 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.12.1", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -1331,7 +1653,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] @@ -1339,6 +1661,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "hashlink" @@ -1361,6 +1688,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" + [[package]] name = "hex" version = "0.4.3" @@ -1499,12 +1832,29 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ + "base64", "bytes", "futures-channel", "futures-core", @@ -1512,7 +1862,9 @@ dependencies = [ "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2 0.6.1", "tokio", @@ -1717,6 +2069,36 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "include_dir" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" +dependencies = [ + "include_dir_macros", +] + +[[package]] +name = "include_dir_macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.12.1" @@ -1725,6 +2107,8 @@ checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] @@ -1742,7 +2126,16 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ - "generic-array", + "generic-array 0.14.7", +] + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", ] [[package]] @@ -1754,6 +2147,15 @@ dependencies = [ "hashbrown 0.15.5", ] +[[package]] +name = "intervaltree" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "270bc34e57047cab801a8c871c124d9dc7132f6473c6401f645524f4e6edd111" +dependencies = [ + "smallvec", +] + [[package]] name = "inventory" version = "0.3.21" @@ -1781,6 +2183,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-macro" version = "0.3.7" @@ -1898,6 +2310,17 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "leaky-bucket" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a396bb213c2d09ed6c5495fd082c991b6ab39c9daf4fff59e6727f85c73e4c5" +dependencies = [ + "parking_lot", + "pin-project-lite", + "tokio", +] + [[package]] name = "libc" version = "0.2.178" @@ -2512,7 +2935,7 @@ dependencies = [ "asynchronous-codec", "bytes", "futures", - "hex", + "hex 0.4.3", "libp2p-core", "libp2p-identity", "libp2p-noise", @@ -2534,7 +2957,7 @@ dependencies = [ "bytes", "futures", "getrandom 0.2.16", - "hex", + "hex 0.4.3", "js-sys", "libp2p-core", "libp2p-identity", @@ -2621,6 +3044,232 @@ dependencies = [ "yamux 0.13.8", ] +[[package]] +name = "libredox" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df15f6eac291ed1cf25865b1ee60399f57e7c227e7f51bdbd4c5270396a9ed50" +dependencies = [ + "bitflags 2.10.0", + "libc", +] + +[[package]] +name = "librqbit" +version = "8.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dadca8f521242010a4c846ef5f224c217009c92e272709cdc08ba9cdabe62983" +dependencies = [ + "anyhow", + "arc-swap", + "async-compression", + "async-stream", + "async-trait", + "backoff", + "base64", + "bincode 2.0.1", + "bitvec", + "byteorder", + "bytes", + "dashmap", + "futures", + "governor", + "hex 0.4.3", + "http", + "intervaltree", + "itertools 0.14.0", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-core", + "librqbit-dht", + "librqbit-peer-protocol", + "librqbit-sha1-wrapper", + "librqbit-tracker-comms", + "librqbit-upnp", + "memmap2", + "mime_guess", + "parking_lot", + "rand 0.9.2", + "regex", + "reqwest", + "rlimit", + "serde", + "serde_json", + "serde_urlencoded", + "serde_with", + "size_format", + "tokio", + "tokio-socks", + "tokio-stream", + "tokio-util", + "tracing", + "url", + "urlencoding", + "uuid", + "walkdir", +] + +[[package]] +name = "librqbit-bencode" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "606dff526ba81e3eca33e2bb28b53afa2bc0b2c41d252333fa44e6c11abb37da" +dependencies = [ + "anyhow", + "bytes", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-sha1-wrapper", + "serde", +] + +[[package]] +name = "librqbit-buffers" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c78b907d6171a7191c162b2b60db46d254ebde6a95282b77372af556c1463" +dependencies = [ + "bytes", + "librqbit-clone-to-owned", + "serde", +] + +[[package]] +name = "librqbit-clone-to-owned" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbd1e66d773ba9c475ff89286dc1d6f9d167cbb898603797467dd0ea6844c445" +dependencies = [ + "bytes", +] + +[[package]] +name = "librqbit-core" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55a02cc6fce6743ad38661ccd6fafc6cf1ae5e0106a9922836b0524dbe752378" +dependencies = [ + "anyhow", + "assert_cfg", + "bytes", + "data-encoding", + "directories", + "hex 0.4.3", + "itertools 0.14.0", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "parking_lot", + "rand 0.9.2", + "serde", + "tokio", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "librqbit-dht" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7cc129194337771a86b0399956c4d9bf1cd97c5f24d14a50be38e170f76a54b" +dependencies = [ + "anyhow", + "backoff", + "byteorder", + "bytes", + "chrono", + "dashmap", + "futures", + "hex 0.4.3", + "indexmap 2.12.1", + "leaky-bucket", + "librqbit-bencode", + "librqbit-clone-to-owned", + "librqbit-core", + "parking_lot", + "rand 0.9.2", + "serde", + "serde_json", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", +] + +[[package]] +name = "librqbit-peer-protocol" +version = "4.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a73129497b500505f33d1dc0426319b6a6a208f13fdfaae56224ab8c2346a773" +dependencies = [ + "anyhow", + "bincode 1.3.3", + "bitvec", + "byteorder", + "bytes", + "itertools 0.14.0", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-clone-to-owned", + "librqbit-core", + "serde", +] + +[[package]] +name = "librqbit-sha1-wrapper" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79373a02db73159e4de7ca5d27b6eeae2d540df66c6801db2b01c5513d087524" +dependencies = [ + "assert_cfg", + "crypto-hash", +] + +[[package]] +name = "librqbit-tracker-comms" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08204944c5be677a5de8e1230e0249fce5c14abef23048e26452c6fb03f1b260" +dependencies = [ + "anyhow", + "async-stream", + "byteorder", + "futures", + "librqbit-bencode", + "librqbit-buffers", + "librqbit-core", + "parking_lot", + "rand 0.9.2", + "reqwest", + "serde", + "tokio", + "tokio-util", + "tracing", + "url", + "urlencoding", +] + +[[package]] +name = "librqbit-upnp" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545aad6124c97201055983137e12a19f34acad565120c3cd30596cbd72e8fa86" +dependencies = [ + "anyhow", + "bstr", + "futures", + "httparse", + "network-interface", + "quick-xml", + "reqwest", + "serde", + "tokio", + "tracing", + "url", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -2705,6 +3354,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memmap2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -2724,12 +3382,38 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mime_guess" +version = "2.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7c44f8e672c00fe5308fa235f821cb4198414e1c77935c1ab6948d3fd78550e" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -2815,6 +3499,23 @@ dependencies = [ "unsigned-varint 0.7.2", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "ndarray" version = "0.17.1" @@ -2822,7 +3523,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7c9125e8f6f10c9da3aad044cc918cf8784fa34de857b1aa68038eb05a50a9" dependencies = [ "matrixmultiply", - "num-complex", + "num-complex 0.4.6", "num-integer", "num-traits", "portable-atomic", @@ -2894,6 +3595,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "network-interface" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e79101e6efcffacab279462884a7eebf65ea5f4ac2cc727b60c715a9aa04722" +dependencies = [ + "cc", + "libc", + "thiserror 2.0.17", + "winapi", +] + [[package]] name = "networking" version = "0.0.1" @@ -2941,6 +3654,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + [[package]] name = "ntapi" version = "0.4.1" @@ -2959,6 +3678,19 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-complex 0.2.4", + "num-integer", + "num-iter", + "num-rational 0.2.4", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -2969,6 +3701,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -2993,6 +3735,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.4.2" @@ -3031,7 +3795,7 @@ checksum = "7aac2e6a6e4468ffa092ad43c39b81c79196c2bb773b8db4085f695efe3bba17" dependencies = [ "libc", "ndarray", - "num-complex", + "num-complex 0.4.6", "num-integer", "num-traits", "pyo3", @@ -3070,6 +3834,56 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "5.1.0" @@ -3231,6 +4045,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "polling" version = "3.11.0" @@ -3367,15 +4187,15 @@ dependencies = [ "bigdecimal", "either", "hashbrown 0.16.1", - "indexmap", + "indexmap 2.12.1", "indoc", "inventory", "libc", "lock_api", "memoffset", "num-bigint", - "num-complex", - "num-rational", + "num-complex 0.4.6", + "num-rational 0.4.2", "num-traits", "once_cell", "ordered-float", @@ -3480,12 +4300,12 @@ dependencies = [ "anyhow", "chrono", "either", - "indexmap", + "indexmap 2.12.1", "inventory", "itertools 0.14.0", "log", "maplit", - "num-complex", + "num-complex 0.4.6", "numpy", "ordered-float", "pyo3", @@ -3501,13 +4321,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2426ba759d848787239d80f9fdb1f223786976f87fb6c3da8188ca7c17744b28" dependencies = [ "heck", - "indexmap", + "indexmap 2.12.1", "proc-macro2", "quote", "rustpython-parser", "syn 2.0.111", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-protobuf" version = "0.8.1" @@ -3530,6 +4365,16 @@ dependencies = [ "unsigned-varint 0.8.0", ] +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.9" @@ -3601,6 +4446,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.7.3" @@ -3701,6 +4552,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "rawpointer" version = "0.2.1" @@ -3755,6 +4615,37 @@ dependencies = [ "bitflags 2.10.0", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 2.0.17", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "regex" version = "1.12.2" @@ -3784,6 +4675,45 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "reqwest" +version = "0.12.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6eff9328d40131d43bd911d42d79eb6a47312002a4daefc9e37f17e74a7701a" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -3814,6 +4744,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rlimit" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7043b63bd0cd1aaa628e476b80e6d4023a3b50eb32789f2728908107bd0c793a" +dependencies = [ + "libc", +] + [[package]] name = "rtnetlink" version = "0.13.1" @@ -4009,6 +4948,48 @@ dependencies = [ "cipher", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4023,12 +5004,35 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der", - "generic-array", + "generic-array 0.14.7", "pkcs8", "subtle", "zeroize", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.10.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -4102,6 +5106,49 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +dependencies = [ + "base64", + "chrono", + "hex 0.4.3", + "indexmap 1.9.3", + "indexmap 2.12.1", + "schemars 0.9.0", + "schemars 1.1.0", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.111", +] + [[package]] name = "sha1" version = "0.10.6" @@ -4168,12 +5215,28 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "siphasher" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +[[package]] +name = "size_format" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed5f6ab2122c6dec69dca18c72fa4590a27e581ad20d44960fe74c032a0b23b" +dependencies = [ + "generic-array 0.12.4", + "num", +] + [[package]] name = "slab" version = "0.4.11" @@ -4238,6 +5301,15 @@ dependencies = [ "sha1", ] +[[package]] +name = "spinning_top" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96d2d1d716fb500937168cc09353ffdc7a012be8475ac7308e1bdf0e3923300" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.7.3" @@ -4294,6 +5366,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -4365,12 +5446,31 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "target-lexicon" version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -4524,6 +5624,40 @@ dependencies = [ "syn 2.0.111", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-socks" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" +dependencies = [ + "either", + "futures-util", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "tokio-util" version = "0.7.17" @@ -4543,7 +5677,7 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ - "indexmap", + "indexmap 2.12.1", "serde_core", "serde_spanned", "toml_datetime", @@ -4576,6 +5710,45 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4655,6 +5828,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "typewit" +version = "1.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8c1ae7cc0fdb8b842d65d127cb981574b0d2b249b74d1c7a2986863dc134f71" + [[package]] name = "uint" version = "0.10.0" @@ -4663,7 +5842,7 @@ checksum = "909988d098b2f738727b161a106cfc7cab00c539c2687a8836f8e565976fb53e" dependencies = [ "byteorder", "crunchy", - "hex", + "hex 0.4.3", "static_assertions", ] @@ -4719,6 +5898,12 @@ dependencies = [ "unic-common", ] +[[package]] +name = "unicase" +version = "2.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -4799,6 +5984,12 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "unty" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" + [[package]] name = "url" version = "2.5.7" @@ -4811,6 +6002,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4853,12 +6050,34 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "virtue" +version = "0.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4947,6 +6166,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.83" @@ -5007,6 +6239,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -5395,6 +6636,15 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x25519-dalek" version = "2.0.1" diff --git a/Cargo.toml b/Cargo.toml index e16c7b67..2daee2ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "3" members = [ "rust/networking", + "rust/downloads", "rust/exo_pyo3_bindings", "rust/system_custodian", "rust/util", @@ -25,6 +26,7 @@ opt-level = 3 [workspace.dependencies] ## Crate members as common dependencies networking = { path = "rust/networking" } +downloads = { path = "rust/downloads" } system_custodian = { path = "rust/system_custodian" } util = { path = "rust/util" } diff --git a/rust/downloads/Cargo.toml b/rust/downloads/Cargo.toml new file mode 100644 index 00000000..cc3a34b8 --- /dev/null +++ b/rust/downloads/Cargo.toml @@ -0,0 +1,40 @@ +[package] +name = "downloads" +version = { workspace = true } +edition = { workspace = true } +publish = false + +[lib] +doctest = false +name = "downloads" +path = "src/lib.rs" + +[lints] +workspace = true + +[dependencies] +# macro dependencies +derive_more = { workspace = true } + +# async +tokio = { workspace = true, features = ["full"] } +futures = { workspace = true } +futures-util = { workspace = true } + +# utility dependencies +util = { workspace = true } +thiserror = { workspace = true } +anyhow = { workspace = true } +itertools = { workspace = true } + +# tracing/logging +log = { workspace = true } + +# BitTorrent library +librqbit = "8.1.1" + +# Embed torrent files +include_dir = "0.7" + +# Serialization +serde = { version = "1.0", features = ["derive"] } diff --git a/rust/downloads/src/bencode.rs b/rust/downloads/src/bencode.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/rust/downloads/src/bencode.rs @@ -0,0 +1 @@ + diff --git a/rust/downloads/src/embedded.rs b/rust/downloads/src/embedded.rs new file mode 100644 index 00000000..c4270c8a --- /dev/null +++ b/rust/downloads/src/embedded.rs @@ -0,0 +1,48 @@ +//! Embedded torrent file access +//! +//! Provides access to .torrent files embedded in the binary at compile time + +use include_dir::{Dir, include_dir}; + +/// Embedded torrent files directory +static TORRENTS: Dir<'_> = include_dir!("$CARGO_MANIFEST_DIR/torrents"); + +/// Get an embedded torrent file by model_id and revision +/// +/// # Arguments +/// * `model_id` - Model identifier (e.g., "mlx-community/Qwen3-30B-A3B-4bit") +/// * `revision` - Git commit hash +/// +/// # Returns +/// The torrent file contents, or None if not found +#[inline] +pub fn get_embedded_torrent(model_id: &str, revision: &str) -> Option> { + let path = format!("{}/{}.torrent", model_id, revision); + TORRENTS + .get_file(&path) + .map(|file| file.contents().to_vec()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_embedded_torrent() { + // Test with the Qwen3 torrent we have + let result = get_embedded_torrent( + "mlx-community/Qwen3-30B-A3B-4bit", + "d388dead1515f5e085ef7a0431dd8fadf0886c57", + ); + + assert!(result.is_some(), "Expected to find embedded torrent"); + let torrent_data = result.unwrap(); + assert!(!torrent_data.is_empty(), "Torrent data should not be empty"); + } + + #[test] + fn test_missing_torrent() { + let result = get_embedded_torrent("nonexistent/model", "abc123"); + assert!(result.is_none(), "Expected None for missing torrent"); + } +} diff --git a/rust/downloads/src/lib.rs b/rust/downloads/src/lib.rs new file mode 100644 index 00000000..4f286a1a --- /dev/null +++ b/rust/downloads/src/lib.rs @@ -0,0 +1,19 @@ +//! BitTorrent-based download system for model shards using rqbit +//! +//! This crate provides: +//! - Torrent session management via rqbit +//! - Embedded torrent file access +//! - Private tracker announce handling +//! - Selective file download based on shard layer ranges + +#![allow(clippy::missing_inline_in_public_items)] + +pub mod bencode; +pub mod embedded; +pub mod progress; +pub mod session; +pub mod tracker; + +pub use embedded::get_embedded_torrent; +pub use session::{DownloadProgress, TorrentSession}; +pub use tracker::handle_announce; diff --git a/rust/downloads/src/progress.rs b/rust/downloads/src/progress.rs new file mode 100644 index 00000000..0f9ed6e0 --- /dev/null +++ b/rust/downloads/src/progress.rs @@ -0,0 +1,77 @@ +//! Download progress tracking +//! +//! Types for tracking and reporting download progress to Python + +use std::collections::HashMap; + +/// Progress update for a torrent download +#[derive(Debug, Clone)] +pub struct DownloadProgress { + /// Total bytes to download + pub total_bytes: u64, + + /// Bytes downloaded so far + pub downloaded_bytes: u64, + + /// Number of pieces completed + pub pieces_completed: usize, + + /// Total number of pieces + pub total_pieces: usize, + + /// Number of peers connected + pub peers_connected: usize, + + /// Download speed in bytes/second + pub speed_bytes_per_sec: f64, + + /// Estimated time remaining in seconds + pub eta_seconds: Option, + + /// Per-file progress + pub files: HashMap, +} + +#[derive(Debug, Clone)] +pub struct FileProgress { + /// Total file size + pub total_bytes: u64, + + /// Bytes downloaded for this file + pub downloaded_bytes: u64, + + /// Whether the file is complete + pub complete: bool, +} + +impl DownloadProgress { + #[inline] + pub fn new(total_bytes: u64, total_pieces: usize) -> Self { + Self { + total_bytes, + downloaded_bytes: 0, + pieces_completed: 0, + total_pieces, + peers_connected: 0, + speed_bytes_per_sec: 0.0, + eta_seconds: None, + files: HashMap::new(), + } + } + + #[inline] + pub fn progress_fraction(&self) -> f64 { + if self.total_bytes == 0 { + 0.0 + } else { + #[allow(clippy::cast_precision_loss)] + let fraction = self.downloaded_bytes as f64 / self.total_bytes as f64; + fraction + } + } + + #[inline] + pub fn is_complete(&self) -> bool { + self.pieces_completed >= self.total_pieces + } +} diff --git a/rust/downloads/src/session.rs b/rust/downloads/src/session.rs new file mode 100644 index 00000000..bc1930d6 --- /dev/null +++ b/rust/downloads/src/session.rs @@ -0,0 +1,162 @@ +//! Torrent session management using rqbit +//! +//! Provides a wrapper around rqbit's Session for managing torrent downloads +//! with persistent seeding and selective file downloads. + +use anyhow::{Context, Result}; +use librqbit::{AddTorrent, AddTorrentOptions, Api, ManagedTorrentHandle, Session, SessionOptions}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::RwLock; + +/// Download progress information +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DownloadProgress { + pub downloaded_bytes: u64, + pub total_bytes: u64, + pub download_speed: f64, + pub upload_speed: f64, + pub peers_connected: usize, + pub is_finished: bool, +} + +/// Torrent session handle for managing multiple torrents +pub struct TorrentSession { + session: Arc, + api: Arc, + session_dir: PathBuf, + torrents: Arc>>, +} + +impl TorrentSession { + /// Create a new torrent session + /// + /// # Arguments + /// * `session_dir` - Directory to store session state and downloaded files + pub async fn new(session_dir: PathBuf) -> Result { + std::fs::create_dir_all(&session_dir).context("Failed to create session directory")?; + + let opts = SessionOptions { + disable_dht: false, + disable_dht_persistence: false, + dht_config: None, + persistence: true, + fastresume: true, + ..Default::default() + }; + + let session = Session::new_with_opts(session_dir.clone(), opts) + .await + .context("Failed to create rqbit session")?; + + let api = Api::new(Arc::clone(&session), None); + + Ok(Self { + session: Arc::new(session), + api: Arc::new(api), + session_dir, + torrents: Arc::new(RwLock::new(HashMap::new())), + }) + } + + /// Add a torrent from raw bytes + /// + /// # Arguments + /// * `torrent_data` - Raw .torrent file contents + /// * `save_path` - Where to save the downloaded files + /// * `file_indices` - Optional list of file indices to download (None = all files) + /// + /// # Returns + /// Info hash as hex string + pub async fn add_torrent( + &self, + torrent_data: Vec, + save_path: PathBuf, + file_indices: Option>, + ) -> Result { + let opts = AddTorrentOptions { + overwrite: false, + only_files_regex: None, + only_files: file_indices + .map(|indices| librqbit::AddTorrentOptions::only_files_from_vec(indices)), + output_folder: Some(save_path.to_string_lossy().to_string()), + ..Default::default() + }; + + let add_torrent = AddTorrent::from_bytes(torrent_data); + + let handle = self + .session + .add_torrent(add_torrent, Some(opts)) + .await + .context("Failed to add torrent")?; + + let info_hash = handle.info_hash().as_string(); + + self.torrents + .write() + .await + .insert(info_hash.clone(), handle); + + Ok(info_hash) + } + + /// Get download progress for a torrent + pub async fn get_progress(&self, info_hash: &str) -> Result { + let torrents = self.torrents.read().await; + let handle = torrents.get(info_hash).context("Torrent not found")?; + + let stats = handle.stats(); + let state = handle.state(); + + Ok(DownloadProgress { + downloaded_bytes: stats.downloaded_bytes, + total_bytes: stats.total_bytes, + download_speed: stats.download_speed, + upload_speed: stats.upload_speed, + peers_connected: stats.peers_connected, + is_finished: state.is_finished(), + }) + } + + /// Wait until torrent download is completed + pub async fn wait_until_completed(&self, info_hash: &str) -> Result<()> { + let torrents = self.torrents.read().await; + let handle = torrents.get(info_hash).context("Torrent not found")?; + + handle + .wait_until_completed() + .await + .context("Failed to wait for completion")?; + + Ok(()) + } + + /// Enable seeding for a completed torrent + /// + /// Note: rqbit seeds by default after completion, this is a no-op + /// but kept for API compatibility + pub async fn enable_seeding(&self, _info_hash: &str) -> Result<()> { + // rqbit automatically seeds after download completion + // This is kept for API compatibility + Ok(()) + } + + /// Remove a torrent from the session + pub async fn remove_torrent(&self, info_hash: &str) -> Result<()> { + let mut torrents = self.torrents.write().await; + + if let Some(handle) = torrents.remove(info_hash) { + drop(handle); + } + + Ok(()) + } + + /// Get list of all torrent info hashes in the session + pub async fn list_torrents(&self) -> Vec { + self.torrents.read().await.keys().cloned().collect() + } +} diff --git a/rust/downloads/src/tracker.rs b/rust/downloads/src/tracker.rs new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/rust/downloads/src/tracker.rs @@ -0,0 +1 @@ + diff --git a/rust/downloads/torrents/mlx-community/Qwen3-30B-A3B-4bit/d388dead1515f5e085ef7a0431dd8fadf0886c57.torrent b/rust/downloads/torrents/mlx-community/Qwen3-30B-A3B-4bit/d388dead1515f5e085ef7a0431dd8fadf0886c57.torrent new file mode 100644 index 00000000..86d6545e Binary files /dev/null and b/rust/downloads/torrents/mlx-community/Qwen3-30B-A3B-4bit/d388dead1515f5e085ef7a0431dd8fadf0886c57.torrent differ diff --git a/rust/exo_pyo3_bindings/Cargo.toml b/rust/exo_pyo3_bindings/Cargo.toml index 12803ab4..5cfd5e5a 100644 --- a/rust/exo_pyo3_bindings/Cargo.toml +++ b/rust/exo_pyo3_bindings/Cargo.toml @@ -23,6 +23,7 @@ workspace = true [dependencies] networking = { workspace = true } +downloads = { workspace = true } # interop pyo3 = { version = "0.27.1", features = [ diff --git a/rust/exo_pyo3_bindings/src/downloads.rs b/rust/exo_pyo3_bindings/src/downloads.rs new file mode 100644 index 00000000..4e1e9663 --- /dev/null +++ b/rust/exo_pyo3_bindings/src/downloads.rs @@ -0,0 +1,327 @@ +//! Downloads module - BitTorrent downloads PyO3 bindings + +use crate::ext::*; +use downloads::bencode::AnnounceParams; +use downloads::tracker::{PeerInfo, TopologyData, handle_announce as rust_handle_announce}; +use downloads::{DownloadProgress, TorrentSession}; +use pyo3::prelude::*; +use pyo3::types::{PyBytes, PyDict}; +use std::net::Ipv4Addr; +use std::path::PathBuf; +use std::sync::Arc; +use tokio::sync::Mutex; + +/// Handle a tracker announce request +/// +/// Args: +/// params: Dictionary with announce parameters (info_hash, peer_id, port, etc.) +/// peers: List of peer dictionaries (node_id, ip, port, has_complete, priority) +/// +/// Returns: +/// Bencoded announce response as bytes +#[pyfunction] +fn handle_tracker_announce( + py: Python<'_>, + params: &Bound<'_, PyDict>, + peers: &Bound<'_, pyo3::types::PyList>, +) -> PyResult> { + // Parse announce params + let info_hash = { + let info_hash_item = params + .get_item("info_hash")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing info_hash"))?; + let info_hash_bytes: &[u8] = info_hash_item.extract()?; + + if info_hash_bytes.len() != 20 { + return Err(pyo3::exceptions::PyValueError::new_err( + "info_hash must be 20 bytes", + )); + } + + let mut info_hash = [0u8; 20]; + info_hash.copy_from_slice(info_hash_bytes); + info_hash + }; + + let peer_id = { + let peer_id_item = params + .get_item("peer_id")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing peer_id"))?; + let peer_id_bytes: &[u8] = peer_id_item.extract()?; + + if peer_id_bytes.len() != 20 { + return Err(pyo3::exceptions::PyValueError::new_err( + "peer_id must be 20 bytes", + )); + } + + let mut peer_id = [0u8; 20]; + peer_id.copy_from_slice(peer_id_bytes); + peer_id + }; + + let port: u16 = params + .get_item("port")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing port"))? + .extract()?; + + let uploaded: u64 = params + .get_item("uploaded")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing uploaded"))? + .extract()?; + + let downloaded: u64 = params + .get_item("downloaded")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing downloaded"))? + .extract()?; + + let left: u64 = params + .get_item("left")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing left"))? + .extract()?; + + let compact: bool = params + .get_item("compact")? + .map(|v| v.extract().unwrap_or(true)) + .unwrap_or(true); + + let announce_params = AnnounceParams { + info_hash, + peer_id, + port, + uploaded, + downloaded, + left, + compact, + event: None, // TODO: parse event if needed + }; + + // Parse peer list + let peer_infos: Result, PyErr> = peers + .iter() + .map(|peer_item| { + let peer_dict: &Bound<'_, PyDict> = peer_item.downcast()?; + let node_id: String = peer_dict + .get_item("node_id")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing node_id"))? + .extract()?; + + let ip_str: String = peer_dict + .get_item("ip")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing ip"))? + .extract()?; + + let ip: Ipv4Addr = ip_str + .parse() + .map_err(|_| pyo3::exceptions::PyValueError::new_err("Invalid IP address"))?; + + let port: u16 = peer_dict + .get_item("port")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("Missing port"))? + .extract()?; + + let has_complete: bool = peer_dict + .get_item("has_complete")? + .map(|v: Bound<'_, pyo3::PyAny>| v.extract().unwrap_or(false)) + .unwrap_or(false); + + let priority: i32 = peer_dict + .get_item("priority")? + .map(|v: Bound<'_, pyo3::PyAny>| v.extract().unwrap_or(0)) + .unwrap_or(0); + + Ok(PeerInfo { + node_id, + ip, + port, + has_complete, + priority, + }) + }) + .collect(); + + let peer_infos = peer_infos?; + + let topology = TopologyData { peers: peer_infos }; + + // Call Rust tracker handler + let response_bytes = rust_handle_announce(&announce_params, &topology).pyerr()?; + + // Return as Python bytes + Ok(PyBytes::new(py, &response_bytes).unbind()) +} + +/// Get an embedded torrent file +/// +/// Args: +/// model_id: Model identifier (e.g., "mlx-community/Qwen3-30B-A3B-4bit") +/// revision: Git commit hash +/// +/// Returns: +/// Torrent file contents as bytes, or None if not found +#[pyfunction] +fn get_embedded_torrent( + py: Python<'_>, + model_id: String, + revision: String, +) -> PyResult>> { + match downloads::get_embedded_torrent(&model_id, &revision) { + Some(data) => Ok(Some(PyBytes::new(py, &data).unbind())), + None => Ok(None), + } +} + +/// Python wrapper for TorrentSession +#[pyclass] +struct TorrentSessionHandle { + session: Arc>, +} + +#[pymethods] +impl TorrentSessionHandle { + /// Create a new torrent session + /// + /// Args: + /// session_dir: Directory to store session state and downloads + #[new] + fn new(session_dir: String) -> PyResult { + let session_path = PathBuf::from(session_dir); + + let session = tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { TorrentSession::new(session_path).await }) + .pyerr()?; + + Ok(Self { + session: Arc::new(Mutex::new(session)), + }) + } + + /// Add a torrent from bytes + /// + /// Args: + /// torrent_data: Raw .torrent file contents + /// save_path: Where to save downloaded files + /// file_indices: Optional list of file indices to download + /// + /// Returns: + /// Info hash as hex string + fn add_torrent( + &self, + py: Python<'_>, + torrent_data: Vec, + save_path: String, + file_indices: Option>, + ) -> PyResult { + let session = Arc::clone(&self.session); + let save_path = PathBuf::from(save_path); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { + session + .lock() + .await + .add_torrent(torrent_data, save_path, file_indices) + .await + }) + .pyerr() + }) + } + + /// Get download progress for a torrent + /// + /// Args: + /// info_hash: Torrent info hash + /// + /// Returns: + /// Dictionary with progress information + fn get_progress(&self, py: Python<'_>, info_hash: String) -> PyResult> { + let session = Arc::clone(&self.session); + + let progress: DownloadProgress = py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { session.lock().await.get_progress(&info_hash).await }) + .pyerr() + })?; + + let dict = PyDict::new(py); + dict.set_item("downloaded_bytes", progress.downloaded_bytes)?; + dict.set_item("total_bytes", progress.total_bytes)?; + dict.set_item("download_speed", progress.download_speed)?; + dict.set_item("upload_speed", progress.upload_speed)?; + dict.set_item("peers_connected", progress.peers_connected)?; + dict.set_item("is_finished", progress.is_finished)?; + + Ok(dict.unbind()) + } + + /// Wait until torrent download is completed + /// + /// Args: + /// info_hash: Torrent info hash + fn wait_until_completed(&self, py: Python<'_>, info_hash: String) -> PyResult<()> { + let session = Arc::clone(&self.session); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { session.lock().await.wait_until_completed(&info_hash).await }) + .pyerr() + }) + } + + /// Enable seeding for a torrent + /// + /// Args: + /// info_hash: Torrent info hash + fn enable_seeding(&self, py: Python<'_>, info_hash: String) -> PyResult<()> { + let session = Arc::clone(&self.session); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { session.lock().await.enable_seeding(&info_hash).await }) + .pyerr() + }) + } + + /// Remove a torrent from the session + /// + /// Args: + /// info_hash: Torrent info hash + fn remove_torrent(&self, py: Python<'_>, info_hash: String) -> PyResult<()> { + let session = Arc::clone(&self.session); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { session.lock().await.remove_torrent(&info_hash).await }) + .pyerr() + }) + } + + /// List all torrents in the session + /// + /// Returns: + /// List of info hashes + fn list_torrents(&self, py: Python<'_>) -> PyResult> { + let session = Arc::clone(&self.session); + + py.allow_threads(|| { + tokio::runtime::Runtime::new() + .pyerr()? + .block_on(async { Ok(session.lock().await.list_torrents().await) }) + }) + } +} + +/// Downloads submodule +pub(crate) fn downloads_submodule(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_function(wrap_pyfunction!(handle_tracker_announce, m)?)?; + m.add_function(wrap_pyfunction!(get_embedded_torrent, m)?)?; + m.add_class::()?; + Ok(()) +} diff --git a/rust/exo_pyo3_bindings/src/lib.rs b/rust/exo_pyo3_bindings/src/lib.rs index 4f591b8c..6a78971f 100644 --- a/rust/exo_pyo3_bindings/src/lib.rs +++ b/rust/exo_pyo3_bindings/src/lib.rs @@ -17,10 +17,12 @@ extern crate core; mod allow_threading; +pub(crate) mod downloads; mod examples; pub(crate) mod networking; pub(crate) mod pylibp2p; +use crate::downloads::downloads_submodule; use crate::networking::networking_submodule; use crate::pylibp2p::ident::ident_submodule; use crate::pylibp2p::multiaddr::multiaddr_submodule; @@ -207,6 +209,7 @@ fn main_module(m: &Bound<'_, PyModule>) -> PyResult<()> { ident_submodule(m)?; multiaddr_submodule(m)?; networking_submodule(m)?; + downloads_submodule(m)?; // top-level constructs // TODO: ... diff --git a/scripts/mktorrent.sh b/scripts/mktorrent.sh new file mode 100755 index 00000000..a6debcbc --- /dev/null +++ b/scripts/mktorrent.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p mktorrent -p python3Packages.huggingface-hub -p git -p git-lfs +set -euo pipefail +set -x + +MODEL="$1" + +mkdir -p "$MODEL" + +# Step 1: Clone/fetch the repo and get the hash of head +mkdir -p "$MODEL" +if test -d "$MODEL/git"; then + # Assert that the origin is correct + git -C "$MODEL/git" fetch +else + git clone "https://huggingface.co/$MODEL" "$MODEL/git" +fi + +HASH=$(git -C "$MODEL/git" rev-parse origin/main) +LARGE_FILES=$(git -C "$MODEL/git" lfs ls-files --all --name-only) + +SMALL_DIR="$MODEL/$HASH-small" +LARGE_DIR="$MODEL/$HASH-large" +mkdir -p "$SMALL_DIR" "$LARGE_DIR" + +# Step 2: Prepare files. Two torrents: one for large files and one for metadata. +git -C "$MODEL/git" archive "$HASH" | tar -x -C "$SMALL_DIR" +echo "$LARGE_FILES" | xargs -I{} rm "$SMALL_DIR/{}" + +echo "$LARGE_FILES" | xargs hf download "$MODEL" --revision "$HASH" --local-dir "$LARGE_DIR" --cache-dir "$(realpath .cache)" --include +if test -d "$LARGE_DIR/.cache"; then + echo ".cache created against our wishes, deleting it..." + rm -r "$LARGE_DIR/.cache" +fi + +# Step 3: Create both torrents +mkdir -p "torrents/$MODEL/" +SMALL_TORRENT_PATH="torrents/$MODEL/${HASH}.small.torrent" +LARGE_TORRENT_PATH="torrents/$MODEL/${HASH}.large.torrent" + +mktorrent "$SMALL_DIR/" --output="$SMALL_TORRENT_PATH" \ + -n "$HASH" \ + --web-seed="https://huggingface.co/$MODEL/raw/" \ + --no-date \ + --announce="udp://tracker.opentrackr.org:1337/announce" + # --private + +mktorrent "$LARGE_DIR/" --output="$LARGE_TORRENT_PATH" \ + -n "$HASH" \ + --web-seed="https://huggingface.co/$MODEL/resolve/" \ + --piece-length=24 \ + --no-date \ + --announce="udp://tracker.opentrackr.org:1337/announce" + # --private + +echo "Successfully created torrent files in:" +echo "$SMALL_TORRENT_PATH" +echo "$LARGE_TORRENT_PATH" diff --git a/src/exo/main.py b/src/exo/main.py index 43d69aa5..a8714d5b 100644 --- a/src/exo/main.py +++ b/src/exo/main.py @@ -35,6 +35,7 @@ class Node: api: API | None node_id: NodeId + enable_torrents: bool _tg: TaskGroup = field(init=False, default_factory=anyio.create_task_group) @classmethod @@ -66,7 +67,8 @@ class Node: worker = Worker( node_id, session_id, - exo_shard_downloader(), + exo_shard_downloader(enable_torrents=args.enable_torrents), + initial_connection_messages=[], connection_message_receiver=router.receiver(topics.CONNECTION_MESSAGES), global_event_receiver=router.receiver(topics.GLOBAL_EVENTS), local_event_sender=router.sender(topics.LOCAL_EVENTS), @@ -74,7 +76,6 @@ class Node: ) else: worker = None - # We start every node with a master master = Master( node_id, @@ -98,7 +99,7 @@ class Node: election_result_sender=er_send, ) - return cls(router, worker, election, er_recv, master, api, node_id) + return cls(router, worker, election, er_recv, master, api, node_id, args.enable_torrents) async def run(self): async with self._tg as tg: @@ -175,7 +176,7 @@ class Node: self.worker = Worker( self.node_id, result.session_id, - exo_shard_downloader(), + exo_shard_downloader(enable_torrents=self.enable_torrents), connection_message_receiver=self.router.receiver( topics.CONNECTION_MESSAGES ), @@ -215,6 +216,7 @@ class Args(CamelCaseModel): api_port: PositiveInt = 52415 tb_only: bool = False no_worker: bool = False + enable_torrents: bool = False @classmethod def parse(cls) -> Self: @@ -256,6 +258,12 @@ class Args(CamelCaseModel): "--no-worker", action="store_true", ) + parser.add_argument( + "--enable-torrents", + action="store_true", + dest="enable_torrents", + help="Enable BitTorrent-based downloads (experimental)", + ) args = parser.parse_args() return cls(**vars(args)) # pyright: ignore[reportAny] - We are intentionally validating here, we can't do it statically diff --git a/src/exo/master/api.py b/src/exo/master/api.py index 30f87e2a..79b6720c 100644 --- a/src/exo/master/api.py +++ b/src/exo/master/api.py @@ -5,9 +5,9 @@ from typing import cast import anyio from anyio import create_task_group from anyio.abc import TaskGroup -from fastapi import FastAPI, HTTPException +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse +from fastapi.responses import Response, StreamingResponse from fastapi.staticfiles import StaticFiles from hypercorn.asyncio import serve # pyright: ignore[reportUnknownVariableType] from hypercorn.config import Config @@ -178,6 +178,7 @@ class API: self.app.post("/bench/chat/completions")(self.bench_chat_completions) self.app.get("/state")(lambda: self.state) self.app.get("/events")(lambda: self._event_log) + self.app.get("/_internal/announce")(self.tracker_announce) async def place_instance(self, payload: PlaceInstanceParams): command = PlaceInstance( @@ -622,6 +623,65 @@ class API: ] ) + async def tracker_announce(self, request: Request) -> Response: + """BitTorrent tracker announce endpoint for private tracker.""" + try: + from exo_pyo3_bindings import handle_tracker_announce # type: ignore + except ImportError as e: + raise HTTPException( + status_code=501, + detail="Torrent support not available (exo_pyo3_bindings not installed)", + ) from e + + # Parse announce parameters from query string + query_params = dict(request.query_params) + + # Extract required parameters + try: + info_hash_hex = query_params.get("info_hash", "") + peer_id_hex = query_params.get("peer_id", "") + + # URL decode and convert to bytes + info_hash = bytes.fromhex(info_hash_hex) if info_hash_hex else b"" + peer_id = bytes.fromhex(peer_id_hex) if peer_id_hex else b"" + + if len(info_hash) != 20 or len(peer_id) != 20: + raise ValueError("info_hash and peer_id must be 20 bytes") + + params = { + "info_hash": info_hash, + "peer_id": peer_id, + "port": int(query_params.get("port", "6881")), + "uploaded": int(query_params.get("uploaded", "0")), + "downloaded": int(query_params.get("downloaded", "0")), + "left": int(query_params.get("left", "0")), + "compact": query_params.get("compact", "1") == "1", + } + except (ValueError, KeyError) as e: + raise HTTPException( + status_code=400, + detail=f"Invalid announce parameters: {e}", + ) from e + + # Build peer list from topology + # TODO: Implement _build_peer_list_from_topology() to extract peers from self.state.topology + peers = [] # For now, return empty peer list + + # Call Rust tracker handler + try: + response_bytes: bytes = handle_tracker_announce(params, peers) # type: ignore + return Response( + content=response_bytes, + media_type="text/plain", + headers={"Content-Type": "text/plain"}, + ) + except Exception as e: + logger.error(f"Tracker announce error: {e}") + raise HTTPException( + status_code=500, + detail=f"Tracker announce failed: {e}", + ) from e + async def run(self): cfg = Config() cfg.bind = f"0.0.0.0:{self.port}" diff --git a/src/exo/shared/types/models.py b/src/exo/shared/types/models.py index e99a7667..a6bf6c62 100644 --- a/src/exo/shared/types/models.py +++ b/src/exo/shared/types/models.py @@ -16,3 +16,4 @@ class ModelMetadata(CamelCaseModel): n_layers: PositiveInt hidden_size: PositiveInt supports_tensor: bool + revision: str | None = None # Git commit hash for torrent lookup diff --git a/src/exo/worker/download/impl_shard_downloader.py b/src/exo/worker/download/impl_shard_downloader.py index 46f55ff9..f045c396 100644 --- a/src/exo/worker/download/impl_shard_downloader.py +++ b/src/exo/worker/download/impl_shard_downloader.py @@ -12,10 +12,17 @@ from exo.worker.download.download_utils import RepoDownloadProgress, download_sh from exo.worker.download.shard_downloader import ShardDownloader -def exo_shard_downloader(max_parallel_downloads: int = 8) -> ShardDownloader: - return SingletonShardDownloader( - CachedShardDownloader(ResumableShardDownloader(max_parallel_downloads)) - ) +def exo_shard_downloader( + max_parallel_downloads: int = 8, enable_torrents: bool = False +) -> ShardDownloader: + if enable_torrents: + from exo.worker.download.torrent_downloader import TorrentShardDownloader + + base = TorrentShardDownloader(max_parallel_downloads) + else: + base = ResumableShardDownloader(max_parallel_downloads) + + return SingletonShardDownloader(CachedShardDownloader(base)) async def build_base_shard(model_id: str) -> ShardMetadata: diff --git a/src/exo/worker/download/torrent_downloader.py b/src/exo/worker/download/torrent_downloader.py new file mode 100644 index 00000000..ea9c0cc0 --- /dev/null +++ b/src/exo/worker/download/torrent_downloader.py @@ -0,0 +1,212 @@ +"""Torrent-based shard downloader using BitTorrent protocol with private tracker. + +This module implements downloading model shards via BitTorrent with: +- Private tracker integration (/_internal/announce endpoint) +- WebSeed (BEP 19) fallback to HTTPS +- Selective file download based on shard layer ranges +- Persistent seeding of downloaded content +""" + +import asyncio +from collections.abc import AsyncIterator +from pathlib import Path +from typing import Callable + +from loguru import logger + +from exo.shared.constants import EXO_MODELS_DIR +from exo.shared.types.worker.shards import ShardMetadata +from exo.worker.download.download_utils import RepoDownloadProgress +from exo.worker.download.shard_downloader import ShardDownloader + + +class TorrentShardDownloader(ShardDownloader): + """Download model shards using BitTorrent with private tracker.""" + + def __init__(self, max_parallel_downloads: int = 8): + self.max_parallel_downloads = max_parallel_downloads + self._progress_callbacks: list[ + Callable[[ShardMetadata, RepoDownloadProgress], None] + ] = [] + + # Initialize TorrentSessionHandle + try: + from exo_pyo3_bindings import TorrentSessionHandle # type: ignore + + session_dir = EXO_MODELS_DIR / "v2" / ".torrent_session" + session_dir.mkdir(parents=True, exist_ok=True) + self.session = TorrentSessionHandle(str(session_dir)) + except ImportError: + logger.error("exo_pyo3_bindings not available, torrent downloads disabled") + self.session = None + + def on_progress( + self, callback: Callable[[ShardMetadata, RepoDownloadProgress], None] + ) -> None: + """Register a progress callback.""" + self._progress_callbacks.append(callback) + + async def ensure_shard( + self, shard: ShardMetadata, config_only: bool = False + ) -> Path: + """Download a model shard using BitTorrent. + + Args: + shard: Shard metadata including model ID and layer range + config_only: If True, only download config files (not implemented for torrents yet) + + Returns: + Path to the downloaded shard directory + + Raises: + RuntimeError: If torrent file is not found or session not initialized + """ + if self.session is None: + raise RuntimeError( + "TorrentSessionHandle not initialized. " + "exo_pyo3_bindings module not available." + ) + + model_id = str(shard.model_meta.model_id) + + # Resolve "main" branch to commit hash + revision = await self._resolve_revision(model_id, "main") + + # Load embedded torrent file + torrent_data = self._load_embedded_torrent(model_id, revision) + if torrent_data is None: + raise RuntimeError( + f"Torrent not found for {model_id}@{revision}. " + f"Expected at: rust/downloads/torrents/{model_id}/{revision}.torrent. " + f"Please add the torrent file or use HTTP downloads (--enable-torrents=False)." + ) + + # Build v2 path: ~/.exo/models/v2/{model_id}/{revision} + save_path = EXO_MODELS_DIR / "v2" / model_id / revision + save_path.mkdir(parents=True, exist_ok=True) + + # Calculate which files to download based on shard layers + # For now, download all files (selective download will be implemented later) + file_indices = None # None means download all files + + # Add torrent and download + logger.info( + f"Adding torrent for {model_id}@{revision} to session, saving to {save_path}" + ) + info_hash = self.session.add_torrent(torrent_data, str(save_path), file_indices) + + # Wait for download with progress reporting + logger.info(f"Starting download for {info_hash}") + while True: + progress_dict = self.session.get_progress(info_hash) + + # Convert dict to RepoDownloadProgress for callbacks + progress = RepoDownloadProgress( + downloaded=progress_dict["downloaded_bytes"], + total=progress_dict["total_bytes"], + ) + self._report_progress(shard, progress) + + if progress_dict["is_finished"]: + logger.info(f"Download completed for {info_hash}") + break + + await asyncio.sleep(0.5) + + # Enable persistent seeding + logger.info(f"Enabling seeding for {info_hash}") + self.session.enable_seeding(info_hash) + + return save_path + + async def get_shard_download_status( + self, + ) -> AsyncIterator[tuple[Path, RepoDownloadProgress]]: + """Get download status for all shards.""" + if self.session is None: + return + + # List all torrents in the session + info_hashes = self.session.list_torrents() + + for info_hash in info_hashes: + try: + progress_dict = self.session.get_progress(info_hash) + progress = RepoDownloadProgress( + downloaded=progress_dict["downloaded_bytes"], + total=progress_dict["total_bytes"], + ) + + # We don't have a straightforward way to map info_hash back to path + # This would require tracking in the session or metadata + # For now, yield empty path + yield (Path(), progress) + except Exception as e: + logger.error(f"Error getting status for {info_hash}: {e}") + + async def get_shard_download_status_for_shard( + self, shard: ShardMetadata + ) -> RepoDownloadProgress: + """Get download status for a specific shard.""" + if self.session is None: + return RepoDownloadProgress(downloaded=0, total=0) + + model_id = str(shard.model_meta.model_id) + revision = await self._resolve_revision(model_id, "main") + + # We would need to track info_hash -> shard mapping + # For now, return empty progress + return RepoDownloadProgress(downloaded=0, total=0) + + async def _resolve_revision(self, model_id: str, branch: str = "main") -> str: + """Resolve branch name to commit hash using HuggingFace API. + + Args: + model_id: Model identifier (e.g., "mlx-community/Qwen3-30B-A3B-4bit") + branch: Branch name (default: "main") + + Returns: + Git commit hash (SHA) + """ + try: + from huggingface_hub import model_info + + info = model_info(model_id, revision=branch) + return str(info.sha) + except Exception as e: + logger.error(f"Failed to resolve revision for {model_id}@{branch}: {e}") + # Fallback to "main" as revision if API call fails + logger.warning(f"Using branch name '{branch}' as revision") + return branch + + def _load_embedded_torrent(self, model_id: str, revision: str) -> bytes | None: + """Load embedded torrent file from Rust binary. + + Args: + model_id: Model identifier + revision: Git commit hash + + Returns: + Torrent file contents, or None if not found + """ + try: + from exo_pyo3_bindings import get_embedded_torrent # type: ignore + + result: bytes | None = get_embedded_torrent(model_id, revision) # type: ignore + return result + except ImportError: + logger.warning("exo_pyo3_bindings not available, cannot load torrents") + return None + except Exception as e: + logger.error(f"Error loading torrent for {model_id}@{revision}: {e}") + return None + + def _report_progress( + self, shard: ShardMetadata, progress: RepoDownloadProgress + ) -> None: + """Report progress to all registered callbacks.""" + for callback in self._progress_callbacks: + try: + callback(shard, progress) + except Exception as e: + logger.error(f"Error in progress callback: {e}")