feat: add mirror handling and OCI mirror type (#553)

This is the start of the "mirror" implemnetation that would

- be able to handle multiple mirrors of a single channel
- reroute a channel to an "internal" mirror (for on-prem deployments)
- handle OCI channels to pull data from

Author: wolfv

Cargo.toml

@@ -48,7 +48,7 @@ chrono = { version = "0.4.35", default-features = false, features = [
     "serde",
     "alloc",
 ] }
-clap = { version = "4.5.1", features = ["derive"] }
+clap = { version = "4.5.2", features = ["derive"] }
 cmake = "0.1.50"
 console = { version = "0.15.8", features = ["windows-console-colors"] }
 criterion = "0.5"
@@ -67,6 +67,7 @@ getrandom = { version = "0.2.12", default-features = false }
 glob = "0.3.1"
 hex = "0.4.3"
 hex-literal = "0.4.1"
+http = "0.2"
 humansize = "2.1.3"
 humantime = "2.1.0"
 indexmap = "2.2.5"
@@ -78,7 +79,7 @@ keyring = "2.3.2"
 lazy-regex = "3.1.0"
 lazy_static = "1.4.0"
 libc = { version = "0.2" }
-libloading = "0.8.2"
+libloading = "0.8.3"
 libz-sys = { version = "1.1.15", default-features = false }
 md-5 = "0.10.6"
 memchr = "2.7.1"
@@ -98,10 +99,11 @@ quote = "1.0.35"
 rand = "0.8.5"
 reflink-copy = "0.1.15"
 regex = "1.10.3"
-reqwest = { version = "0.11.24", default-features = false }
+reqwest = { version = "0.11.25", default-features = false }
 reqwest-middleware = "0.2.4"
+reqwest-retry = "0.4.0"
 resolvo = { version = "0.4.0" }
-retry-policies = { version = "0.2.1", default-features = false }
+retry-policies = { version = "0.3.0", default-features = false }
 rstest = { version = "0.18.2" }
 rstest_reuse = "0.6.0"
 serde = { version = "1.0.197" }
@@ -122,7 +124,7 @@ smallvec = { version = "1.13.1", features = [
 strum = { version = "0.26.1", features = ["derive"] }
 superslice = "1.0.0"
 syn = "2.0.52"
-sysinfo = "0.30.6"
+sysinfo = "0.30.7"
 tar = "0.4.40"
 task-local-extensions = "0.1.4"
 tempdir = "0.3.7"

crates/rattler/src/package_cache.rs

@@ -5,6 +5,7 @@ use chrono::Utc;
 use fxhash::FxHashMap;
 use itertools::Itertools;
 use rattler_conda_types::{package::ArchiveIdentifier, PackageRecord};
+use rattler_digest::Sha256Hash;
 use rattler_networking::retry_policies::{DoNotRetryPolicy, RetryDecision, RetryPolicy};
 use rattler_package_streaming::ExtractError;
 use reqwest::StatusCode;
@@ -38,6 +39,14 @@ pub struct CacheKey {
     name: String,
     version: String,
     build_string: String,
+    sha256: Option<Sha256Hash>,
+}
+
+impl CacheKey {
+    /// Return the sha256 hash of the package if it is known.
+    pub fn sha256(&self) -> Option<Sha256Hash> {
+        self.sha256
+    }
 }
 
 impl From<ArchiveIdentifier> for CacheKey {
@@ -46,6 +55,7 @@ impl From<ArchiveIdentifier> for CacheKey {
             name: pkg.name,
             version: pkg.version,
             build_string: pkg.build_string,
+            sha256: None,
         }
     }
 }
@@ -56,6 +66,7 @@ impl From<&PackageRecord> for CacheKey {
             name: record.name.as_normalized().to_string(),
             version: record.version.to_string(),
             build_string: record.build.clone(),
+            sha256: record.sha256,
         }
     }
 }
@@ -200,7 +211,10 @@ impl PackageCache {
         client: reqwest_middleware::ClientWithMiddleware,
         retry_policy: impl RetryPolicy + Send + 'static,
     ) -> Result<PathBuf, PackageCacheError> {
-        self.get_or_fetch(pkg, move |destination| async move {
+        let request_start = Utc::now();
+        let cache_key = pkg.into();
+        let sha256 = cache_key.sha256();
+        self.get_or_fetch(cache_key, move |destination| async move {
             let mut current_try = 0;
             loop {
                 current_try += 1;
@@ -209,6 +223,7 @@ impl PackageCache {
                     client.clone(),
                     url.clone(),
                     &destination,
+                    sha256,
                 )
                 .await;
 
@@ -230,7 +245,7 @@ impl PackageCache {
                 }
 
                 // Determine whether or not to retry based on the retry policy
-                let execute_after = match retry_policy.should_retry(current_try) {
+                let execute_after = match retry_policy.should_retry(request_start, current_try) {
                     RetryDecision::Retry { execute_after } => execute_after,
                     RetryDecision::DoNotRetry => return Err(err),
                 };

crates/rattler_networking/Cargo.toml

@@ -11,15 +11,18 @@ license.workspace = true
 readme.workspace = true
 
 [features]
+default = ["native-tls"]
 native-tls = ['reqwest/native-tls']
 rustls-tls = ['reqwest/rustls-tls']
 
 [dependencies]
 anyhow = { workspace = true }
 async-trait = { workspace = true }
 base64 = { workspace = true }
+chrono = { workspace = true }
 dirs = { workspace = true }
 fslock = { workspace = true }
+http = { workspace = true }
 itertools = { workspace = true }
 keyring = { workspace = true }
 lazy_static = { workspace = true }
@@ -44,3 +47,6 @@ anyhow = { workspace = true }
 insta = { workspace = true, features = ["json"] }
 tempfile = { workspace = true }
 tokio = { workspace = true, features = ["macros"] }
+axum = { workspace = true }
+reqwest-retry = { workspace = true }
+sha2 = { workspace = true }

crates/rattler_networking/src/lib.rs

@@ -2,11 +2,14 @@
 
 //! Networking utilities for Rattler, specifically authenticating requests
 pub use authentication_middleware::AuthenticationMiddleware;
-
 pub use authentication_storage::{authentication::Authentication, storage::AuthenticationStorage};
+pub use mirror_middleware::MirrorMiddleware;
+pub use oci_middleware::OciMiddleware;
 
 pub mod authentication_middleware;
 pub mod authentication_storage;
+pub mod mirror_middleware;
+pub mod oci_middleware;
 pub mod retry_policies;
 
 mod redaction;