diff --git a/Cargo.lock b/Cargo.lock index b01abd83..408a4be8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,6 +30,12 @@ dependencies = [ "zerocopy 0.7.35", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -89,18 +95,387 @@ dependencies = [ "syn", ] -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-config" +version = "1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8191fb3091fa0561d1379ef80333c3c7191c6f0435d986e85821bcf7acbd1126" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http 0.2.12", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", +] + +[[package]] +name = "aws-credential-types" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", +] + +[[package]] +name = "aws-runtime" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] + +[[package]] +name = "aws-sdk-s3" +version = "1.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43fad71130014e11f42fadbdcce5df12ee61866f8ab9bad773b138d4b3c11087" +dependencies = [ + "ahash", + "aws-credential-types", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "fastrand", + "hex", + "hmac", + "http 0.2.12", + "http-body 0.4.6", + "lru", + "once_cell", + "percent-encoding", + "regex-lite", + "sha2", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "1.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b90cfe6504115e13c41d3ea90286ede5aa14da294f3fe077027a6e83850843c" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167c0fad1f212952084137308359e8e4c4724d1c643038ce163f06de9662c1d0" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-sts" +version = "1.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cb5f98188ec1435b68097daa2a37d74b9d17c9caa799466338a8d1544e71b9d" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sigv4" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" +dependencies = [ + "aws-credential-types", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "crypto-bigint 0.5.5", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.1.0", + "once_cell", + "p256", + "percent-encoding", + "ring", + "sha2", + "subtle", + "time", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-async" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "aws-smithy-checksums" +version = "0.60.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598b1689d001c4d4dc3cb386adb07d37786783aee3ac4b324bcadac116bf3d23" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http 0.2.12", + "http-body 0.4.6", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + +[[package]] +name = "aws-smithy-eventstream" +version = "0.60.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cef7d0a272725f87e51ba2bf89f8c21e4df61b9e49ae1ac367a6d69916ef7c90" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + +[[package]] +name = "aws-smithy-http" +version = "0.60.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http-body 0.4.6", + "once_cell", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] + +[[package]] +name = "aws-smithy-json" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" +dependencies = [ + "aws-smithy-types", +] + +[[package]] +name = "aws-smithy-query" +version = "0.60.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2fbd61ceb3fe8a1cb7352e42689cec5335833cd9f94103a61e98f9bb61c64bb" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + +[[package]] +name = "aws-smithy-runtime" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1ce695746394772e7000b39fe073095db6d45a862d0767dd5ad0ac0d7f8eb87" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "h2", + "http 0.2.12", + "http-body 0.4.6", + "http-body 1.0.1", + "httparse", + "hyper", + "hyper-rustls", + "once_cell", + "pin-project-lite", + "pin-utils", + "rustls", + "tokio", + "tracing", +] + +[[package]] +name = "aws-smithy-runtime-api" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e086682a53d3aa241192aa110fa8dfce98f2f5ac2ead0de84d41582c7e8fdb96" +dependencies = [ + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.1.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.1.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -116,18 +491,46 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base16ct" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" + [[package]] name = "base32" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "022dfe9eb35f19ebbcb51e0b40a5ab759f46ad60cadf7297e0bd085afb50e076" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + [[package]] name = "bit-set" version = "0.5.3" @@ -179,6 +582,16 @@ dependencies = [ "serde", ] +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.1.7" @@ -203,9 +616,15 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.6", + "windows-targets", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "core-foundation" version = "0.9.4" @@ -222,6 +641,55 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "cpufeatures" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-bigint" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +dependencies = [ + "generic-array", + "rand_core", + "subtle", + "zeroize", +] + +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -267,6 +735,16 @@ dependencies = [ "syn", ] +[[package]] +name = "der" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "deranged" version = "0.3.11" @@ -291,6 +769,19 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", +] + +[[package]] +name = "ecdsa" +version = "0.14.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +dependencies = [ + "der", + "elliptic-curve", + "rfc6979", + "signature", ] [[package]] @@ -299,6 +790,26 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +[[package]] +name = "elliptic-curve" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +dependencies = [ + "base16ct", + "crypto-bigint 0.4.9", + "der", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -321,6 +832,16 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "ff" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +dependencies = [ + "rand_core", + "subtle", +] + [[package]] name = "fnv" version = "1.0.7" @@ -452,18 +973,29 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "group" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +dependencies = [ + "ff", + "rand_core", + "subtle", +] + [[package]] name = "h2" -version = "0.4.5" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ - "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http", + "futures-util", + "http 0.2.12", "indexmap 2.2.6", "slab", "tokio", @@ -482,6 +1014,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "heck" @@ -507,6 +1043,26 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.1.0" @@ -518,6 +1074,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -525,7 +1092,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.1.0", ] [[package]] @@ -536,8 +1103,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http", - "http-body", + "http 1.1.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -547,6 +1114,12 @@ version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "humantime" version = "2.1.0" @@ -555,60 +1128,42 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.4.1" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", + "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "httparse", + "httpdate", "itoa", "pin-project-lite", - "smallvec", + "socket2", "tokio", + "tower-service", + "tracing", "want", ] [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", + "http 0.2.12", "hyper", - "hyper-util", + "log", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" -dependencies = [ - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "pin-project-lite", - "socket2", - "tokio", - "tower", - "tower-service", - "tracing", ] [[package]] @@ -641,8 +1196,11 @@ dependencies = [ "async-recursion", "async-stream", "async-trait", + "aws-config", + "aws-credential-types", + "aws-sdk-s3", "base32", - "base64", + "base64 0.22.1", "bytes", "chrono", "futures", @@ -724,12 +1282,6 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" -[[package]] -name = "ipnet" -version = "2.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" - [[package]] name = "itertools" version = "0.13.0" @@ -794,6 +1346,15 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lru" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "md-5" version = "0.10.6" @@ -819,12 +1380,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - [[package]] name = "miniz_oxide" version = "0.7.4" @@ -852,6 +1407,15 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -878,22 +1442,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25a0c4b3a0e31f8b66f71ad8064521efa773910196e2cde791436f13409f3b45" dependencies = [ "async-trait", - "base64", "bytes", "chrono", "futures", "humantime", - "hyper", "itertools", - "md-5", "parking_lot", "percent-encoding", - "quick-xml", - "rand", - "reqwest", - "ring", - "serde", - "serde_json", "snafu", "tokio", "tracing", @@ -913,6 +1468,23 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + +[[package]] +name = "p256" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" +dependencies = [ + "ecdsa", + "elliptic-curve", + "sha2", +] + [[package]] name = "parking_lot" version = "0.12.3" @@ -933,7 +1505,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -942,31 +1514,11 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pin-project" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project-lite" @@ -980,6 +1532,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs8" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eca2c590a5f85da82668fa685c09ce2888b9430e83299debf1f34b65fd4a4ba" +dependencies = [ + "der", + "spki", +] + [[package]] name = "portable-atomic" version = "1.7.0" @@ -1132,16 +1694,6 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" -[[package]] -name = "quick-xml" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "quick_cache" version = "0.6.9" @@ -1154,54 +1706,6 @@ dependencies = [ "parking_lot", ] -[[package]] -name = "quinn" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" -dependencies = [ - "bytes", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", -] - -[[package]] -name = "quinn-proto" -version = "0.11.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" -dependencies = [ - "bytes", - "rand", - "ring", - "rustc-hash", - "rustls", - "slab", - "thiserror", - "tinyvec", - "tracing", -] - -[[package]] -name = "quinn-udp" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" -dependencies = [ - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.52.0", -] - [[package]] name = "quote" version = "1.0.36" @@ -1259,6 +1763,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + [[package]] name = "regex-syntax" version = "0.8.4" @@ -1266,48 +1776,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] -name = "reqwest" -version = "0.12.5" +name = "rfc6979" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pemfile", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", - "winreg", + "crypto-bigint 0.4.9", + "hmac", + "zeroize", ] [[package]] @@ -1366,10 +1842,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] -name = "rustc-hash" -version = "2.0.0" +name = "rustc_version" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] [[package]] name = "rustix" @@ -1386,55 +1865,44 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.21.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ - "once_cell", + "log", "ring", - "rustls-pki-types", "rustls-webpki", - "subtle", - "zeroize", + "sct", ] [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" dependencies = [ "openssl-probe", "rustls-pemfile", - "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", - "rustls-pki-types", + "base64 0.21.7", ] -[[package]] -name = "rustls-pki-types" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" - [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ "ring", - "rustls-pki-types", "untrusted", ] @@ -1480,6 +1948,30 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "sec1" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "2.11.1" @@ -1503,6 +1995,12 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.210" @@ -1544,25 +2042,13 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - [[package]] name = "serde_with" version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", @@ -1586,6 +2072,47 @@ dependencies = [ "syn", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "signature" +version = "1.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +dependencies = [ + "digest", + "rand_core", +] + [[package]] name = "slab" version = "0.4.9" @@ -1638,6 +2165,16 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spki" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67cf02bbac7a337dc36e4f5a693db6c21e7863f45070f7064577eb4367a3212b" +dependencies = [ + "base64ct", + "der", +] + [[package]] name = "strsim" version = "0.11.1" @@ -1684,12 +2221,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" - [[package]] name = "target-lexicon" version = "0.12.16" @@ -1798,6 +2329,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.52.0", @@ -1816,12 +2348,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ "rustls", - "rustls-pki-types", "tokio", ] @@ -1838,27 +2369,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "pin-project", - "pin-project-lite", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - [[package]] name = "tower-service" version = "0.3.3" @@ -1958,12 +2468,30 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "uuid" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" + [[package]] name = "version_check" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wait-timeout" version = "0.2.0" @@ -2023,18 +2551,6 @@ dependencies = [ "wasm-bindgen-shared", ] -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "wasm-bindgen-macro" version = "0.2.92" @@ -2064,29 +2580,6 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" -[[package]] -name = "wasm-streams" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "winapi-util" version = "0.1.9" @@ -2102,16 +2595,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", + "windows-targets", ] [[package]] @@ -2120,7 +2604,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] @@ -2129,22 +2613,7 @@ version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows-targets", ] [[package]] @@ -2153,46 +2622,28 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -2205,48 +2656,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -2254,14 +2681,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winreg" -version = "0.52.0" +name = "xmlparser" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" [[package]] name = "yansi" diff --git a/Justfile b/Justfile index 0fdfb0fc..b120b1cf 100644 --- a/Justfile +++ b/Justfile @@ -3,7 +3,7 @@ alias pre := pre-commit # run all tests test *args='': - AWS_ALLOW_HTTP=1 AWS_ENDPOINT_URL=http://localhost:9000 AWS_ACCESS_KEY_ID=minio123 AWS_SECRET_ACCESS_KEY=minio123 cargo test {{args}} + cargo test --all {{args}} # compile but don't run all tests compile-tests *args='': diff --git a/README.md b/README.md index 786befca..99f1af2e 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ just test This is just an alias for ``` -AWS_ALLOW_HTTP=1 AWS_ENDPOINT_URL=http://localhost:9000 AWS_ACCESS_KEY_ID=minio123 AWS_SECRET_ACCESS_KEY=minio123 cargo test +cargo test --all ``` > [!TIP] diff --git a/compose.yaml b/compose.yaml index 6034aac4..5f3ee303 100644 --- a/compose.yaml +++ b/compose.yaml @@ -4,7 +4,7 @@ volumes: services: minio: container_name: icechunk_minio - image: quay.io/minio/minio + image: minio/minio entrypoint: | /bin/sh -c ' for bucket in testbucket externalbucket arraylake-repo-bucket diff --git a/icechunk-python/src/lib.rs b/icechunk-python/src/lib.rs index c1f2bce1..0be1b6b1 100644 --- a/icechunk-python/src/lib.rs +++ b/icechunk-python/src/lib.rs @@ -112,8 +112,10 @@ type KeyRanges = Vec<(String, (Option, Option))>; impl PyIcechunkStore { async fn store_exists(storage: StorageConfig) -> PyIcechunkStoreResult { - let storage = - storage.make_cached_storage().map_err(PyIcechunkStoreError::UnkownError)?; + let storage = storage + .make_cached_storage() + .await + .map_err(PyIcechunkStoreError::UnkownError)?; let exists = Repository::exists(storage.as_ref()).await?; Ok(exists) } diff --git a/icechunk-python/src/storage.rs b/icechunk-python/src/storage.rs index 22b1ea18..5217572f 100644 --- a/icechunk-python/src/storage.rs +++ b/icechunk-python/src/storage.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; use icechunk::{ storage::{ - object_store::{S3Config, S3Credentials}, + s3::{S3Config, S3Credentials}, virtual_ref::ObjectStoreVirtualChunkResolverConfig, }, zarr::StorageConfig, diff --git a/icechunk-python/tests/test_distributed_writers.py b/icechunk-python/tests/test_distributed_writers.py index 81abb828..fb464e41 100644 --- a/icechunk-python/tests/test_distributed_writers.py +++ b/icechunk-python/tests/test_distributed_writers.py @@ -81,6 +81,7 @@ async def test_distributed_writers(): "bucket": "testbucket", "prefix": "python-distributed-writers-test__" + str(time.time()), "endpoint_url": "http://localhost:9000", + "region": "us-east-1", "allow_http": True, } store_config = {"inline_chunk_threshold_bytes": 5} diff --git a/icechunk-python/tests/test_virtual_ref.py b/icechunk-python/tests/test_virtual_ref.py index 1264d517..dbcb63cb 100644 --- a/icechunk-python/tests/test_virtual_ref.py +++ b/icechunk-python/tests/test_virtual_ref.py @@ -49,6 +49,7 @@ async def test_write_minino_virtual_refs(): ), endpoint_url="http://localhost:9000", allow_http=True, + region="us-east-1", ), mode="r+", config=StoreConfig( @@ -59,6 +60,7 @@ async def test_write_minino_virtual_refs(): ), endpoint_url="http://localhost:9000", allow_http=True, + region="us-east-1", ) ), ) diff --git a/icechunk/Cargo.toml b/icechunk/Cargo.toml index 6c3fc497..52713ee2 100644 --- a/icechunk/Cargo.toml +++ b/icechunk/Cargo.toml @@ -11,7 +11,7 @@ bytes = { version = "1.7.2", features = ["serde"] } base64 = "0.22.1" futures = "0.3.30" itertools = "0.13.0" -object_store = { version = "0.11.0", features = ["aws"] } +object_store = { version = "0.11.0" } rand = "0.8.5" thiserror = "1.0.64" serde_json = "1.0.128" @@ -28,6 +28,9 @@ rmp-serde = "1.3.0" url = "2.5.2" async-stream = "0.3.5" rmpv = { version = "1.3.0", features = ["serde", "with-serde"] } +aws-sdk-s3 = "1.53.0" +aws-config = "1.5.7" +aws-credential-types = "1.2.1" [dev-dependencies] pretty_assertions = "1.4.1" diff --git a/icechunk/src/refs.rs b/icechunk/src/refs.rs index 68d8016f..e5ba906c 100644 --- a/icechunk/src/refs.rs +++ b/icechunk/src/refs.rs @@ -46,7 +46,7 @@ pub enum RefError { pub type RefResult = Result; -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Hash)] pub enum Ref { Tag(String), Branch(String), @@ -196,7 +196,7 @@ async fn branch_history<'a, 'b>( branch: &'b str, ) -> RefResult> + 'a> { let key = branch_root(branch)?; - let all = storage.ref_versions(key.as_str()).await; + let all = storage.ref_versions(key.as_str()).await?; Ok(all.map_err(|e| e.into()).and_then(move |version_id| async move { let version = version_id .strip_suffix(".json") diff --git a/icechunk/src/storage/caching.rs b/icechunk/src/storage/caching.rs index 1800ec45..e36f08b9 100644 --- a/icechunk/src/storage/caching.rs +++ b/icechunk/src/storage/caching.rs @@ -152,7 +152,10 @@ impl Storage for MemCachingStorage { self.backend.write_ref(ref_key, overwrite_refs, bytes).await } - async fn ref_versions(&self, ref_name: &str) -> BoxStream> { + async fn ref_versions( + &self, + ref_name: &str, + ) -> StorageResult>> { self.backend.ref_versions(ref_name).await } } diff --git a/icechunk/src/storage/logging.rs b/icechunk/src/storage/logging.rs index 7e5334b1..38ec82f3 100644 --- a/icechunk/src/storage/logging.rs +++ b/icechunk/src/storage/logging.rs @@ -121,7 +121,10 @@ impl Storage for LoggingStorage { self.backend.write_ref(ref_key, overwrite_refs, bytes).await } - async fn ref_versions(&self, ref_name: &str) -> BoxStream> { + async fn ref_versions( + &self, + ref_name: &str, + ) -> StorageResult>> { self.backend.ref_versions(ref_name).await } } diff --git a/icechunk/src/storage/mod.rs b/icechunk/src/storage/mod.rs index f8a9f10d..716602dc 100644 --- a/icechunk/src/storage/mod.rs +++ b/icechunk/src/storage/mod.rs @@ -1,6 +1,15 @@ +use aws_sdk_s3::{ + config::http::HttpResponse, + error::SdkError, + operation::{ + get_object::GetObjectError, list_objects_v2::ListObjectsV2Error, + put_object::PutObjectError, + }, + primitives::ByteStreamError, +}; use core::fmt; use futures::stream::BoxStream; -use std::sync::Arc; +use std::{ffi::OsString, sync::Arc}; use async_trait::async_trait; use bytes::Bytes; @@ -12,6 +21,7 @@ pub mod caching; pub mod logging; pub mod object_store; +pub mod s3; pub mod virtual_ref; pub use caching::MemCachingStorage; @@ -19,30 +29,36 @@ pub use object_store::ObjectStorage; use crate::format::{ attributes::AttributesTable, manifest::Manifest, snapshot::Snapshot, AttributesId, - ByteRange, ChunkId, ManifestId, Path, SnapshotId, + ByteRange, ChunkId, ManifestId, SnapshotId, }; #[derive(Debug, Error)] pub enum StorageError { #[error("error contacting object store {0}")] ObjectStore(#[from] ::object_store::Error), + #[error("bad object store prefix {0:?}")] + BadPrefix(OsString), + #[error("error getting object from object store {0}")] + S3GetObjectError(#[from] SdkError), + #[error("error writing object to object store {0}")] + S3PutObjectError(#[from] SdkError), + #[error("error listing objects in object store {0}")] + S3ListObjectError(#[from] SdkError), + #[error("error streaming bytes from object store {0}")] + S3StreamError(#[from] ByteStreamError), #[error("messagepack decode error: {0}")] MsgPackDecodeError(#[from] rmp_serde::decode::Error), #[error("messagepack encode error: {0}")] MsgPackEncodeError(#[from] rmp_serde::encode::Error), - #[error("error parsing RecordBatch from parquet file {0}.")] - BadRecordBatchRead(Path), #[error("cannot overwrite ref: {0}")] RefAlreadyExists(String), #[error("ref not found: {0}")] RefNotFound(String), - #[error("generic storage error: {0}")] - OtherError(#[from] Arc), #[error("unknown storage error: {0}")] Other(String), } -type StorageResult = Result; +pub type StorageResult = Result; /// Fetch and write the parquet files that represent the repository in object store /// @@ -77,7 +93,10 @@ pub trait Storage: fmt::Debug { async fn get_ref(&self, ref_key: &str) -> StorageResult; async fn ref_names(&self) -> StorageResult>; - async fn ref_versions(&self, ref_name: &str) -> BoxStream>; + async fn ref_versions( + &self, + ref_name: &str, + ) -> StorageResult>>; async fn write_ref( &self, ref_key: &str, diff --git a/icechunk/src/storage/object_store.rs b/icechunk/src/storage/object_store.rs index 7374e9df..56742e70 100644 --- a/icechunk/src/storage/object_store.rs +++ b/icechunk/src/storage/object_store.rs @@ -5,17 +5,12 @@ use crate::format::{ }; use async_trait::async_trait; use bytes::Bytes; -use core::fmt; use futures::{stream::BoxStream, StreamExt, TryStreamExt}; use object_store::{ - aws::{AmazonS3Builder, S3ConditionalPut}, - local::LocalFileSystem, - memory::InMemory, - path::Path as ObjectPath, - Attribute, AttributeValue, Attributes, GetOptions, GetRange, ObjectStore, PutMode, - PutOptions, PutPayload, + local::LocalFileSystem, memory::InMemory, path::Path as ObjectPath, Attribute, + AttributeValue, Attributes, GetOptions, GetRange, ObjectStore, PutMode, PutOptions, + PutPayload, }; -use serde::{Deserialize, Serialize}; use std::{ fs::create_dir_all, future::ready, ops::Bound, path::Path as StdPath, sync::Arc, }; @@ -61,58 +56,7 @@ const MANIFEST_PREFIX: &str = "manifests/"; const CHUNK_PREFIX: &str = "chunks/"; const REF_PREFIX: &str = "refs"; -#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] -pub struct S3Credentials { - pub access_key_id: String, - pub secret_access_key: String, - pub session_token: Option, -} - -#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] -pub struct S3Config { - pub region: Option, - pub endpoint: Option, - pub credentials: Option, - pub allow_http: Option, -} - -// TODO: Hide this behind a feature flag? -impl S3Config { - pub fn to_builder(&self) -> AmazonS3Builder { - let builder = if let Some(credentials) = &self.credentials { - let builder = AmazonS3Builder::new() - .with_access_key_id(credentials.access_key_id.clone()) - .with_secret_access_key(credentials.secret_access_key.clone()); - - if let Some(token) = &credentials.session_token { - builder.with_token(token.clone()) - } else { - builder - } - } else { - AmazonS3Builder::from_env() - }; - - let builder = if let Some(region) = &self.region { - builder.with_region(region.clone()) - } else { - builder - }; - - let builder = if let Some(endpoint) = &self.endpoint { - builder.with_endpoint(endpoint.clone()) - } else { - builder - }; - - if let Some(allow_http) = self.allow_http { - builder.with_allow_http(allow_http) - } else { - builder - } - } -} - +#[derive(Debug)] pub struct ObjectStorage { store: Arc, prefix: String, @@ -157,24 +101,6 @@ impl ObjectStorage { }) } - pub fn new_s3_store( - bucket_name: impl Into, - prefix: impl Into, - config: Option, - ) -> Result { - let config = config.unwrap_or_default(); - let builder = config.to_builder(); - let builder = builder.with_conditional_put(S3ConditionalPut::ETagMatch); - let store = builder.with_bucket_name(bucket_name.into()).build()?; - Ok(ObjectStorage { - store: Arc::new(store), - prefix: prefix.into(), - artificially_sort_refs_in_mem: false, - supports_create_if_not_exists: true, - supports_metadata: true, - }) - } - fn get_path( &self, file_prefix: &str, @@ -225,11 +151,6 @@ impl ObjectStorage { } } -impl fmt::Debug for ObjectStorage { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "ObjectStorage, prefix={}, store={}", self.prefix, self.store) - } -} #[async_trait] impl Storage for ObjectStorage { async fn fetch_snapshot( @@ -391,7 +312,10 @@ impl Storage for ObjectStorage { .collect()) } - async fn ref_versions(&self, ref_name: &str) -> BoxStream> { + async fn ref_versions( + &self, + ref_name: &str, + ) -> StorageResult>> { let res = self.do_ref_versions(ref_name).await; if self.artificially_sort_refs_in_mem { #[allow(clippy::expect_used)] @@ -401,9 +325,9 @@ impl Storage for ObjectStorage { let mut all = res.try_collect::>().await.expect("Error fetching ref versions"); all.sort(); - futures::stream::iter(all.into_iter().map(Ok)).boxed() + Ok(futures::stream::iter(all.into_iter().map(Ok)).boxed()) } else { - res + Ok(res) } } diff --git a/icechunk/src/storage/s3.rs b/icechunk/src/storage/s3.rs new file mode 100644 index 00000000..d4955a1c --- /dev/null +++ b/icechunk/src/storage/s3.rs @@ -0,0 +1,419 @@ +use std::{ops::Bound, path::PathBuf, sync::Arc}; + +use async_stream::try_stream; +use async_trait::async_trait; +use aws_config::{meta::region::RegionProviderChain, AppName, BehaviorVersion}; +use aws_credential_types::Credentials; +use aws_sdk_s3::{ + config::{Builder, Region}, + error::ProvideErrorMetadata, + primitives::ByteStream, + Client, +}; +use bytes::Bytes; +use futures::StreamExt; +use serde::{Deserialize, Serialize}; + +use crate::{ + format::{ + attributes::AttributesTable, format_constants, manifest::Manifest, + snapshot::Snapshot, AttributesId, ByteRange, ChunkId, FileTypeTag, ManifestId, + SnapshotId, + }, + zarr::ObjectId, + Storage, StorageError, +}; + +use super::StorageResult; + +#[derive(Debug)] +pub struct S3Storage { + client: Arc, + prefix: String, + bucket: String, +} + +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +pub struct S3Credentials { + pub access_key_id: String, + pub secret_access_key: String, + pub session_token: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +pub struct S3Config { + pub region: Option, + pub endpoint: Option, + pub credentials: Option, + pub allow_http: Option, +} + +pub async fn mk_client(config: Option<&S3Config>) -> Client { + let region = config + .and_then(|c| c.region.as_ref()) + .map(|r| RegionProviderChain::first_try(Some(Region::new(r.clone())))) + .unwrap_or_else(RegionProviderChain::default_provider); + + let endpoint = config.and_then(|c| c.endpoint.clone()); + let allow_http = config.and_then(|c| c.allow_http).unwrap_or(false); + let credentials = config.and_then(|c| c.credentials.clone()); + #[allow(clippy::unwrap_used)] + let app_name = AppName::new("icechunk").unwrap(); + let mut aws_config = aws_config::defaults(BehaviorVersion::v2024_03_28()) + .region(region) + .app_name(app_name); + + if let Some(endpoint) = endpoint { + aws_config = aws_config.endpoint_url(endpoint) + } + + if let Some(credentials) = credentials { + aws_config = aws_config.credentials_provider(Credentials::new( + credentials.access_key_id, + credentials.secret_access_key, + credentials.session_token, + None, + "user", + )); + } + + let mut s3_builder = Builder::from(&aws_config.load().await); + + if allow_http { + s3_builder = s3_builder.force_path_style(true); + } + + let config = s3_builder.build(); + + Client::from_conf(config) +} + +const SNAPSHOT_PREFIX: &str = "snapshots/"; +const MANIFEST_PREFIX: &str = "manifests/"; +// const ATTRIBUTES_PREFIX: &str = "attributes/"; +const CHUNK_PREFIX: &str = "chunks/"; +const REF_PREFIX: &str = "refs"; + +impl S3Storage { + pub async fn new_s3_store( + bucket_name: impl Into, + prefix: impl Into, + config: Option<&S3Config>, + ) -> Result { + let client = Arc::new(mk_client(config).await); + Ok(S3Storage { client, prefix: prefix.into(), bucket: bucket_name.into() }) + } + + fn get_path( + &self, + file_prefix: &str, + id: &ObjectId, + ) -> StorageResult { + // we serialize the url using crockford + let path = PathBuf::from_iter([ + self.prefix.as_str(), + file_prefix, + id.to_string().as_str(), + ]); + path.into_os_string().into_string().map_err(StorageError::BadPrefix) + } + + fn get_snapshot_path(&self, id: &SnapshotId) -> StorageResult { + self.get_path(SNAPSHOT_PREFIX, id) + } + + fn get_manifest_path(&self, id: &ManifestId) -> StorageResult { + self.get_path(MANIFEST_PREFIX, id) + } + + fn get_chunk_path(&self, id: &ChunkId) -> StorageResult { + self.get_path(CHUNK_PREFIX, id) + } + + fn ref_key(&self, ref_key: &str) -> StorageResult { + let path = PathBuf::from_iter([self.prefix.as_str(), REF_PREFIX, ref_key]); + path.into_os_string().into_string().map_err(StorageError::BadPrefix) + } + + async fn get_object(&self, key: &str) -> StorageResult { + Ok(self + .client + .get_object() + .bucket(self.bucket.clone()) + .key(key) + .send() + .await? + .body + .collect() + .await? + .into_bytes()) + } + + async fn get_object_range( + &self, + key: &str, + range: &ByteRange, + ) -> StorageResult { + let mut b = self.client.get_object().bucket(self.bucket.clone()).key(key); + + if let Some(header) = range_to_header(range) { + b = b.range(header) + }; + + Ok(b.send().await?.body.collect().await?.into_bytes()) + } + + async fn put_object< + I: IntoIterator, impl Into)>, + >( + &self, + key: &str, + content_type: Option>, + metadata: I, + bytes: impl Into, + ) -> StorageResult<()> { + let mut b = self.client.put_object().bucket(self.bucket.clone()).key(key); + + if let Some(ct) = content_type { + b = b.content_type(ct) + }; + + for (k, v) in metadata { + b = b.metadata(k, v); + } + + b.body(bytes.into()).send().await?; + Ok(()) + } +} + +pub fn range_to_header(range: &ByteRange) -> Option { + match range { + ByteRange(Bound::Unbounded, Bound::Unbounded) => None, + ByteRange(Bound::Included(start), Bound::Excluded(end)) => { + Some(format!("bytes={}-{}", start, end - 1)) + } + ByteRange(Bound::Included(start), Bound::Unbounded) => { + Some(format!("bytes={}-", start)) + } + ByteRange(Bound::Included(start), Bound::Included(end)) => { + Some(format!("bytes={}-{}", start, end)) + } + ByteRange(Bound::Excluded(start), Bound::Excluded(end)) => { + Some(format!("bytes={}-{}", start + 1, end - 1)) + } + ByteRange(Bound::Excluded(start), Bound::Unbounded) => { + Some(format!("bytes={}-", start + 1)) + } + ByteRange(Bound::Excluded(start), Bound::Included(end)) => { + Some(format!("bytes={}-{}", start + 1, end)) + } + ByteRange(Bound::Unbounded, Bound::Excluded(end)) => { + Some(format!("bytes=0-{}", end - 1)) + } + ByteRange(Bound::Unbounded, Bound::Included(end)) => { + Some(format!("bytes=0-{}", end)) + } + } +} + +#[async_trait] +impl Storage for S3Storage { + async fn fetch_snapshot(&self, id: &SnapshotId) -> StorageResult> { + let key = self.get_snapshot_path(id)?; + let bytes = self.get_object(key.as_str()).await?; + // TODO: optimize using from_read + let res = rmp_serde::from_slice(bytes.as_ref())?; + Ok(Arc::new(res)) + } + + async fn fetch_attributes( + &self, + _id: &AttributesId, + ) -> StorageResult> { + todo!() + } + + async fn fetch_manifests(&self, id: &ManifestId) -> StorageResult> { + let key = self.get_manifest_path(id)?; + let bytes = self.get_object(key.as_str()).await?; + // TODO: optimize using from_read + let res = rmp_serde::from_slice(bytes.as_ref())?; + Ok(Arc::new(res)) + } + + async fn fetch_chunk(&self, id: &ChunkId, range: &ByteRange) -> StorageResult { + let key = self.get_chunk_path(id)?; + let bytes = self.get_object_range(key.as_str(), range).await?; + Ok(bytes) + } + + async fn write_snapshot( + &self, + id: SnapshotId, + snapshot: Arc, + ) -> StorageResult<()> { + let key = self.get_snapshot_path(&id)?; + let bytes = rmp_serde::to_vec(snapshot.as_ref())?; + let metadata = [( + format_constants::LATEST_ICECHUNK_SNAPSHOT_VERSION_METADATA_KEY, + snapshot.icechunk_snapshot_format_version.to_string(), + )]; + self.put_object( + key.as_str(), + Some(format_constants::LATEST_ICECHUNK_SNAPSHOT_CONTENT_TYPE), + metadata, + bytes, + ) + .await + } + + async fn write_attributes( + &self, + _id: AttributesId, + _table: Arc, + ) -> StorageResult<()> { + todo!() + } + + async fn write_manifests( + &self, + id: ManifestId, + manifest: Arc, + ) -> Result<(), StorageError> { + let key = self.get_manifest_path(&id)?; + let bytes = rmp_serde::to_vec(manifest.as_ref())?; + let metadata = [( + format_constants::LATEST_ICECHUNK_MANIFEST_VERSION_METADATA_KEY, + manifest.icechunk_manifest_format_version.to_string(), + )]; + self.put_object( + key.as_str(), + Some(format_constants::LATEST_ICECHUNK_MANIFEST_CONTENT_TYPE), + metadata, + bytes, + ) + .await + } + + async fn write_chunk( + &self, + id: ChunkId, + bytes: bytes::Bytes, + ) -> Result<(), StorageError> { + let key = self.get_chunk_path(&id)?; + //FIXME: use multipart upload + let metadata: [(String, String); 0] = []; + self.put_object(key.as_str(), None::, metadata, bytes).await + } + + async fn get_ref(&self, ref_key: &str) -> StorageResult { + let key = self.ref_key(ref_key)?; + let res = self + .client + .get_object() + .bucket(self.bucket.clone()) + .key(key.clone()) + .send() + .await; + + match res { + Ok(res) => Ok(res.body.collect().await?.into_bytes()), + Err(err) + if err + .as_service_error() + .map(|e| e.is_no_such_key()) + .unwrap_or(false) => + { + Err(StorageError::RefNotFound(key.to_string())) + } + Err(err) => Err(err.into()), + } + } + + async fn ref_names(&self) -> StorageResult> { + let prefix = self.ref_key("")?; + let mut paginator = self + .client + .list_objects_v2() + .bucket(self.bucket.clone()) + .prefix(prefix.clone()) + .delimiter("/") + .into_paginator() + .send(); + + let mut res = Vec::new(); + + while let Some(page) = paginator.try_next().await? { + for common_prefix in page.common_prefixes() { + if let Some(key) = common_prefix + .prefix() + .as_ref() + .and_then(|key| key.strip_prefix(prefix.as_str())) + .and_then(|key| key.strip_suffix('/')) + { + res.push(key.to_string()); + } + } + } + + Ok(res) + } + + async fn ref_versions( + &self, + ref_name: &str, + ) -> StorageResult>> { + let prefix = self.ref_key(ref_name)?; + let mut paginator = self + .client + .list_objects_v2() + .bucket(self.bucket.clone()) + .prefix(prefix.clone()) + .into_paginator() + .send(); + + let prefix = prefix + "/"; + let stream = try_stream! { + while let Some(page) = paginator.try_next().await? { + for object in page.contents() { + if let Some(key) = object.key.as_ref().and_then(|key| key.strip_prefix(prefix.as_str())) { + yield key.to_string() + } + } + } + }; + Ok(stream.boxed()) + } + + async fn write_ref( + &self, + ref_key: &str, + overwrite_refs: bool, + bytes: Bytes, + ) -> StorageResult<()> { + let key = self.ref_key(ref_key)?; + let mut builder = + self.client.put_object().bucket(self.bucket.clone()).key(key.clone()); + + if !overwrite_refs { + builder = builder.if_none_match("*") + } + + let res = builder.body(bytes.into()).send().await; + + match res { + Ok(_) => Ok(()), + Err(err) => { + let code = err.as_service_error().and_then(|e| e.code()).unwrap_or(""); + if code.contains("PreconditionFailed") + || code.contains("ConditionalRequestConflict") + { + Err(StorageError::RefAlreadyExists(key)) + } else { + Err(err.into()) + } + } + } + } +} diff --git a/icechunk/src/storage/virtual_ref.rs b/icechunk/src/storage/virtual_ref.rs index e4b378bf..8e60257a 100644 --- a/icechunk/src/storage/virtual_ref.rs +++ b/icechunk/src/storage/virtual_ref.rs @@ -1,19 +1,18 @@ use crate::format::manifest::{VirtualChunkLocation, VirtualReferenceError}; use crate::format::ByteRange; use async_trait::async_trait; +use aws_sdk_s3::Client; use bytes::Bytes; use object_store::local::LocalFileSystem; use object_store::{path::Path as ObjectPath, GetOptions, GetRange, ObjectStore}; use serde::{Deserialize, Serialize}; use std::cmp::{max, min}; -use std::collections::HashMap; use std::fmt::Debug; use std::ops::Bound; -use std::sync::Arc; -use tokio::sync::RwLock; -use url; +use tokio::sync::OnceCell; +use url::{self, Url}; -use super::object_store::S3Config; +use super::s3::{mk_client, range_to_header, S3Config}; #[async_trait] pub trait VirtualChunkResolver: Debug { @@ -24,23 +23,85 @@ pub trait VirtualChunkResolver: Debug { ) -> Result; } -#[derive(PartialEq, Eq, Hash, Clone, Debug)] -struct StoreCacheKey(String, String); - #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] pub enum ObjectStoreVirtualChunkResolverConfig { S3(S3Config), } -#[derive(Debug, Default)] +#[derive(Debug)] pub struct ObjectStoreVirtualChunkResolver { - stores: RwLock>>, - config: Option, + s3: OnceCell, + config: Box>, } impl ObjectStoreVirtualChunkResolver { pub fn new(config: Option) -> Self { - Self { stores: RwLock::new(HashMap::new()), config } + Self { s3: Default::default(), config: Box::new(config) } + } + + async fn s3(&self) -> &Client { + let config = self.config.clone(); + self.s3 + .get_or_init(|| async move { + match config.as_ref() { + Some(ObjectStoreVirtualChunkResolverConfig::S3(config)) => { + mk_client(Some(config)).await + } + None => mk_client(None).await, + } + }) + .await + } + + async fn fetch_file( + &self, + url: &Url, + range: &ByteRange, + ) -> Result { + let store = LocalFileSystem::new(); + let options = + GetOptions { range: Option::::from(range), ..Default::default() }; + let path = ObjectPath::parse(url.path()) + .map_err(|e| VirtualReferenceError::OtherError(Box::new(e)))?; + + store + .get_opts(&path, options) + .await + .map_err(|e| VirtualReferenceError::FetchError(Box::new(e)))? + .bytes() + .await + .map_err(|e| VirtualReferenceError::FetchError(Box::new(e))) + } + + async fn fetch_s3( + &self, + url: &Url, + range: &ByteRange, + ) -> Result { + let bucket_name = if let Some(host) = url.host_str() { + host.to_string() + } else { + Err(VirtualReferenceError::CannotParseBucketName( + "No bucket name found".to_string(), + ))? + }; + + let key = url.path(); + let key = key.strip_prefix('/').unwrap_or(key); + let mut b = self.s3().await.get_object().bucket(bucket_name).key(key); + + if let Some(header) = range_to_header(range) { + b = b.range(header) + }; + + Ok(b.send() + .await + .map_err(|e| VirtualReferenceError::FetchError(Box::new(e)))? + .body + .collect() + .await + .map_err(|e| VirtualReferenceError::FetchError(Box::new(e)))? + .into_bytes()) } } @@ -84,79 +145,13 @@ impl VirtualChunkResolver for ObjectStoreVirtualChunkResolver { let VirtualChunkLocation::Absolute(location) = location; let parsed = url::Url::parse(location).map_err(VirtualReferenceError::CannotParseUrl)?; - let path = ObjectPath::parse(parsed.path()) - .map_err(|e| VirtualReferenceError::OtherError(Box::new(e)))?; let scheme = parsed.scheme(); - let bucket_name = if let Some(host) = parsed.host_str() { - host.to_string() - } else if scheme == "file" { - // Host is not required for file scheme, if it is not there, - // we can assume the bucket name is empty and it is a local file - "".to_string() - } else { - Err(VirtualReferenceError::CannotParseBucketName( - "No bucket name found".to_string(), - ))? - }; - - let cache_key = StoreCacheKey(scheme.into(), bucket_name); - - let options = - GetOptions { range: Option::::from(range), ..Default::default() }; - let store = { - let stores = self.stores.read().await; - stores.get(&cache_key).cloned() - }; - let store = match store { - Some(store) => store, - None => { - let new_store: Arc = match scheme { - "file" => { - let fs = LocalFileSystem::new(); - Arc::new(fs) - } - // FIXME: allow configuring auth for virtual references - "s3" => { - let config = if let Some( - ObjectStoreVirtualChunkResolverConfig::S3(config), - ) = &self.config - { - config.clone() - } else { - S3Config::default() - }; - - let s3 = config - .to_builder() - .with_bucket_name(&cache_key.1) - .build() - .map_err(|e| { - VirtualReferenceError::FetchError(Box::new(e)) - })?; - - Arc::new(s3) - } - _ => { - Err(VirtualReferenceError::UnsupportedScheme(scheme.to_string()))? - } - }; - { - self.stores - .write() - .await - .insert(cache_key.clone(), Arc::clone(&new_store)); - } - new_store - } - }; - Ok(store - .get_opts(&path, options) - .await - .map_err(|e| VirtualReferenceError::FetchError(Box::new(e)))? - .bytes() - .await - .map_err(|e| VirtualReferenceError::FetchError(Box::new(e)))?) + match scheme { + "file" => self.fetch_file(&parsed, range).await, + "s3" => self.fetch_s3(&parsed, range).await, + _ => Err(VirtualReferenceError::UnsupportedScheme(scheme.to_string())), + } } } diff --git a/icechunk/src/zarr.rs b/icechunk/src/zarr.rs index 9c73209f..fda8aef8 100644 --- a/icechunk/src/zarr.rs +++ b/icechunk/src/zarr.rs @@ -33,7 +33,8 @@ use crate::{ StorageTransformer, UserAttributes, ZarrArrayMetadata, }, storage::{ - object_store::S3Config, virtual_ref::ObjectStoreVirtualChunkResolverConfig, + s3::{S3Config, S3Storage}, + virtual_ref::ObjectStoreVirtualChunkResolverConfig, }, ObjectStorage, Repository, RepositoryBuilder, SnapshotMetadata, Storage, }; @@ -59,7 +60,7 @@ pub enum StorageConfig { } impl StorageConfig { - pub fn make_storage(&self) -> Result, String> { + pub async fn make_storage(&self) -> Result, String> { match self { StorageConfig::InMemory { prefix } => { Ok(Arc::new(ObjectStorage::new_in_memory_store(prefix.clone()))) @@ -70,15 +71,18 @@ impl StorageConfig { Ok(Arc::new(storage)) } StorageConfig::S3ObjectStore { bucket, prefix, config } => { - let storage = ObjectStorage::new_s3_store(bucket, prefix, config.clone()) + let storage = S3Storage::new_s3_store(bucket, prefix, config.as_ref()) + .await .map_err(|e| format!("Error creating storage: {e}"))?; Ok(Arc::new(storage)) } } } - pub fn make_cached_storage(&self) -> Result, String> { - let storage = self.make_storage()?; + pub async fn make_cached_storage( + &self, + ) -> Result, String> { + let storage = self.make_storage().await?; let cached_storage = Repository::add_in_mem_asset_caching(storage); Ok(cached_storage) } @@ -266,7 +270,7 @@ impl Store { consolidated: &ConsolidatedStore, mode: AccessMode, ) -> Result { - let storage = consolidated.storage.make_cached_storage()?; + let storage = consolidated.storage.make_cached_storage().await?; let (repository, branch) = consolidated.repository.make_repository(storage).await?; Ok(Self::from_repository(repository, mode, branch, consolidated.config.clone())) @@ -1238,7 +1242,7 @@ mod tests { use std::borrow::BorrowMut; - use crate::storage::object_store::S3Credentials; + use crate::storage::s3::S3Credentials; use super::*; use pretty_assertions::assert_eq; diff --git a/icechunk/tests/test_distributed_writes.rs b/icechunk/tests/test_distributed_writes.rs index cd00ea91..f602b2c4 100644 --- a/icechunk/tests/test_distributed_writes.rs +++ b/icechunk/tests/test_distributed_writes.rs @@ -6,30 +6,33 @@ use icechunk::{ format::{ByteRange, ChunkIndices, Path, SnapshotId}, metadata::{ChunkKeyEncoding, ChunkShape, DataType, FillValue}, repository::{get_chunk, ChangeSet, ZarrArrayMetadata}, - storage::object_store::{S3Config, S3Credentials}, - ObjectStorage, Repository, Storage, + storage::s3::{S3Config, S3Credentials, S3Storage}, + Repository, Storage, }; use tokio::task::JoinSet; const SIZE: usize = 10; -fn mk_storage( +async fn mk_storage( prefix: &str, ) -> Result, Box> { - let storage: Arc = Arc::new(ObjectStorage::new_s3_store( - "testbucket", - prefix, - Some(S3Config { - region: None, - endpoint: Some("http://localhost:9000".to_string()), - credentials: Some(S3Credentials { - access_key_id: "minio123".into(), - secret_access_key: "minio123".into(), - session_token: None, + let storage: Arc = Arc::new( + S3Storage::new_s3_store( + "testbucket", + prefix, + Some(&S3Config { + region: Some("us-east-1".to_string()), + endpoint: Some("http://localhost:9000".to_string()), + credentials: Some(S3Credentials { + access_key_id: "minio123".into(), + secret_access_key: "minio123".into(), + session_token: None, + }), + allow_http: Some(true), }), - allow_http: Some(true), - }), - )?); + ) + .await?, + ); Ok(Repository::add_in_mem_asset_caching(storage)) } @@ -109,10 +112,10 @@ async fn verify( async fn test_distributed_writes() -> Result<(), Box> { let prefix = format!("test_distributed_writes_{}", SnapshotId::random()); - let storage1 = mk_storage(prefix.as_str())?; - let storage2 = mk_storage(prefix.as_str())?; - let storage3 = mk_storage(prefix.as_str())?; - let storage4 = mk_storage(prefix.as_str())?; + let storage1 = mk_storage(prefix.as_str()).await?; + let storage2 = mk_storage(prefix.as_str()).await?; + let storage3 = mk_storage(prefix.as_str()).await?; + let storage4 = mk_storage(prefix.as_str()).await?; let mut repo1 = mk_repo(storage1, true).await?; let zarr_meta = ZarrArrayMetadata { @@ -180,7 +183,7 @@ async fn test_distributed_writes() -> Result<(), Box StorageResult { + S3Storage::new_s3_store( + "testbucket", + "test_s3_storage__".to_string() + Utc::now().to_rfc3339().as_str(), + Some(&S3Config { + region: Some("us-east-1".to_string()), + endpoint: Some("http://localhost:9000".to_string()), + credentials: Some(S3Credentials { + access_key_id: "minio123".into(), + secret_access_key: "minio123".into(), + session_token: None, + }), + allow_http: Some(true), + }), + ) + .await +} + +#[tokio::test] +pub async fn test_snapshot_write_read() -> Result<(), Box> { + let storage = mk_storage().await?; + let id = SnapshotId::random(); + let snapshot = Arc::new(Snapshot::empty()); + storage.write_snapshot(id.clone(), snapshot.clone()).await?; + let back = storage.fetch_snapshot(&id).await?; + assert_eq!(snapshot, back); + Ok(()) +} + +#[tokio::test] +pub async fn test_manifest_write_read() -> Result<(), Box> { + let storage = mk_storage().await?; + let id = ManifestId::random(); + let manifest = Arc::new(Manifest::default()); + storage.write_manifests(id.clone(), manifest.clone()).await?; + let back = storage.fetch_manifests(&id).await?; + assert_eq!(manifest, back); + Ok(()) +} + +#[tokio::test] +pub async fn test_chunk_write_read() -> Result<(), Box> { + let storage = mk_storage().await?; + let id = ChunkId::random(); + let bytes = Bytes::from_static(b"hello"); + storage.write_chunk(id.clone(), bytes.clone()).await?; + let back = storage.fetch_chunk(&id, &ByteRange::ALL).await?; + assert_eq!(bytes, back); + + let back = + storage.fetch_chunk(&id, &ByteRange::from_offset_with_length(1, 2)).await?; + assert_eq!(Bytes::from_static(b"el"), back); + + let back = storage.fetch_chunk(&id, &ByteRange::from_offset(1)).await?; + assert_eq!(Bytes::from_static(b"ello"), back); + + let back = storage.fetch_chunk(&id, &ByteRange::to_offset(3)).await?; + assert_eq!(Bytes::from_static(b"hel"), back); + + let back = storage.fetch_chunk(&id, &ByteRange::bounded(1, 4)).await?; + assert_eq!(Bytes::from_static(b"ell"), back); + Ok(()) +} + +#[tokio::test] +pub async fn test_tag_write_get() -> Result<(), Box> { + let storage = mk_storage().await?; + let id = SnapshotId::random(); + create_tag(&storage, "mytag", id.clone(), false).await?; + let back = fetch_tag(&storage, "mytag").await?; + assert_eq!(id, back.snapshot); + Ok(()) +} + +#[tokio::test] +pub async fn test_fetch_non_existing_tag() -> Result<(), Box> { + let storage = mk_storage().await?; + let id = SnapshotId::random(); + create_tag(&storage, "mytag", id.clone(), false).await?; + + let back = fetch_tag(&storage, "non-existing-tag").await; + assert!(matches!(back, Err(RefError::RefNotFound(r)) if r == "non-existing-tag")); + Ok(()) +} + +#[tokio::test] +pub async fn test_create_existing_tag() -> Result<(), Box> { + let storage = mk_storage().await?; + let id = SnapshotId::random(); + create_tag(&storage, "mytag", id.clone(), false).await?; + + let res = create_tag(&storage, "mytag", id.clone(), false).await; + assert!(matches!(res, Err(RefError::TagAlreadyExists(r)) if r == "mytag")); + Ok(()) +} + +#[tokio::test] +pub async fn test_branch_initialization() -> Result<(), Box> { + let storage = mk_storage().await?; + let id = SnapshotId::random(); + + let res = update_branch(&storage, "some-branch", id.clone(), None, false).await?; + assert_eq!(res.0, 0); + + let res = fetch_branch_tip(&storage, "some-branch").await?; + assert_eq!(res.snapshot, id); + + Ok(()) +} + +#[tokio::test] +pub async fn test_fetch_non_existing_branch() -> Result<(), Box> { + let storage = mk_storage().await?; + let id = SnapshotId::random(); + update_branch(&storage, "some-branch", id.clone(), None, false).await?; + + let back = fetch_branch_tip(&storage, "non-existing-branch").await; + assert!(matches!(back, Err(RefError::RefNotFound(r)) if r == "non-existing-branch")); + Ok(()) +} + +#[tokio::test] +pub async fn test_branch_update() -> Result<(), Box> { + let storage = mk_storage().await?; + let id1 = SnapshotId::random(); + let id2 = SnapshotId::random(); + let id3 = SnapshotId::random(); + + let res = update_branch(&storage, "some-branch", id1.clone(), None, false).await?; + assert_eq!(res.0, 0); + + let res = + update_branch(&storage, "some-branch", id2.clone(), Some(&id1), false).await?; + assert_eq!(res.0, 1); + + let res = + update_branch(&storage, "some-branch", id3.clone(), Some(&id2), false).await?; + assert_eq!(res.0, 2); + + let res = fetch_branch_tip(&storage, "some-branch").await?; + assert_eq!(res.snapshot, id3); + + Ok(()) +} + +#[tokio::test] +pub async fn test_ref_names() -> Result<(), Box> { + let storage = mk_storage().await?; + let id1 = SnapshotId::random(); + let id2 = SnapshotId::random(); + update_branch(&storage, "main", id1.clone(), None, false).await?; + update_branch(&storage, "main", id2.clone(), Some(&id1), false).await?; + update_branch(&storage, "foo", id1.clone(), None, false).await?; + update_branch(&storage, "bar", id1.clone(), None, false).await?; + create_tag(&storage, "my-tag", id1.clone(), false).await?; + create_tag(&storage, "my-other-tag", id1.clone(), false).await?; + + let res: HashSet<_> = HashSet::from_iter(list_refs(&storage).await?); + assert_eq!( + res, + HashSet::from_iter([ + Ref::Tag("my-tag".to_string()), + Ref::Tag("my-other-tag".to_string()), + Ref::Branch("main".to_string()), + Ref::Branch("foo".to_string()), + Ref::Branch("bar".to_string()), + ]) + ); + Ok(()) +} diff --git a/icechunk/tests/test_virtual_refs.rs b/icechunk/tests/test_virtual_refs.rs index e5ff2e3b..dc4e6c50 100644 --- a/icechunk/tests/test_virtual_refs.rs +++ b/icechunk/tests/test_virtual_refs.rs @@ -9,7 +9,8 @@ mod tests { metadata::{ChunkKeyEncoding, ChunkShape, DataType, FillValue}, repository::{get_chunk, ChunkPayload, ZarrArrayMetadata}, storage::{ - object_store::{S3Config, S3Credentials}, + s3::{mk_client, S3Config, S3Credentials, S3Storage}, + virtual_ref::ObjectStoreVirtualChunkResolverConfig, ObjectStorage, }, zarr::AccessMode, @@ -25,8 +26,27 @@ mod tests { }; use pretty_assertions::assert_eq; + fn s3_config() -> S3Config { + S3Config { + region: Some("us-east-1".to_string()), + endpoint: Some("http://localhost:9000".to_string()), + credentials: Some(S3Credentials { + access_key_id: "minio123".into(), + secret_access_key: "minio123".into(), + session_token: None, + }), + allow_http: Some(true), + } + } + async fn create_repository(storage: Arc) -> Repository { - Repository::init(storage, true).await.expect("building repository failed").build() + Repository::init(storage, true) + .await + .expect("building repository failed") + .with_virtual_ref_config(ObjectStoreVirtualChunkResolverConfig::S3( + s3_config(), + )) + .build() } async fn write_chunks_to_store( @@ -57,20 +77,12 @@ mod tests { async fn create_minio_repository() -> Repository { let storage: Arc = Arc::new( - ObjectStorage::new_s3_store( + S3Storage::new_s3_store( "testbucket".to_string(), format!("{:?}", ChunkId::random()), - Some(S3Config { - region: None, - endpoint: Some("http://localhost:9000".to_string()), - credentials: Some(S3Credentials { - access_key_id: "minio123".into(), - secret_access_key: "minio123".into(), - session_token: None, - }), - allow_http: Some(true), - }), + Some(&s3_config()), ) + .await .expect("Creating minio storage failed"), ); @@ -84,19 +96,19 @@ mod tests { } async fn write_chunks_to_minio(chunks: impl Iterator) { - use object_store::aws::AmazonS3Builder; - let bucket_name = "testbucket".to_string(); + let client = mk_client(Some(&s3_config())).await; - let store = AmazonS3Builder::new() - .with_access_key_id("minio123") - .with_secret_access_key("minio123") - .with_endpoint("http://localhost:9000") - .with_allow_http(true) - .with_bucket_name(bucket_name) - .build() - .expect("building S3 store failed"); - - write_chunks_to_store(store, chunks).await; + let bucket_name = "testbucket".to_string(); + for (key, bytes) in chunks { + client + .put_object() + .bucket(bucket_name.clone()) + .key(key) + .body(bytes.into()) + .send() + .await + .unwrap(); + } } #[tokio::test(flavor = "multi_thread")] @@ -111,7 +123,7 @@ mod tests { write_chunks_to_local_fs(chunks.iter().cloned()).await; let repo_dir = TempDir::new()?; - let mut ds = create_local_repository(&repo_dir.path()).await; + let mut ds = create_local_repository(repo_dir.path()).await; let zarr_meta = ZarrArrayMetadata { shape: vec![1, 1, 2],