feat: merge pixi-build branch (#950)

* feat: lock file update for source dependencies (#866)

Co-authored-by: Bas Zalmstra <zalmstra.bas@gmail.com>
Co-authored-by: Bas Zalmstra <bas@prefix.dev>

* feat: add input hash to conda package

* feat: add input hash/glob to lock file (#868)

* fix: mark records as changed if hash Option value is different (#874)

This adds the case that where the hash changed, which happened for my
source build, that a re-install is triggered. I think this is the
correct behavior, not sure if it will trigger *too* many re-installs but
did not feel that would be the case.

* fix: python bindings issues (#872)

* fix: normalize channel in lock file (#871)

* fix: dead-lock issue

* feat: use sha in cache lock file, needed for source builds. (#900)

* fix: package hash revalidation (#904)

* refactor!: lock-file API (#942)

This PR significantly refactors the lock-file API to make the
distinction between source and binary conda packages. It also changes
the way we handle borrowing data from the lock-file, preferring
borrowing over owned objects.

The Python bindings have also been completely refactored to be more
pythonic. Instead of having a single `LockedPackage` there is no a class
hierarchy of `LockedPackage` -> `CondaLockedPackage`,
`PypiLockedPackage` -> `CondaLockedSourcePackage` and
`CondaLockedBinaryPackage`.

---------

Co-authored-by: Tim de Jager <tdejager89@gmail.com>
Co-authored-by: Tim de Jager <tim@prefix.dev>
Co-authored-by: nichmor <nmorkotilo@gmail.com>

Author: 3 people

Cargo.toml

@@ -121,6 +121,7 @@ simd-json = { version = "0.14.2", features = ["serde_impl"] }
 serde = { version = "1.0.214" }
 serde_json = { version = "1.0.132" }
 serde_repr = "0.1"
+serde-value = "0.7.0"
 serde_with = "3.11.0"
 serde_yaml = "0.9.34"
 serde-untagged = "0.1.6"

crates/file_url/src/lib.rs

@@ -9,7 +9,8 @@ use std::path::PathBuf;
 use std::str::FromStr;
 use thiserror::Error;
 use typed_path::{
-    Utf8TypedComponent, Utf8TypedPath, Utf8UnixComponent, Utf8WindowsComponent, Utf8WindowsPrefix,
+    Utf8TypedComponent, Utf8TypedPath, Utf8TypedPathBuf, Utf8UnixComponent, Utf8WindowsComponent,
+    Utf8WindowsPrefix,
 };
 use url::{Host, Url};
 
@@ -33,14 +34,7 @@ fn is_windows_drive_letter_segment(segment: &str) -> Option<String> {
     None
 }
 
-/// Tries to convert a `file://` based URL to a path.
-///
-/// We assume that any passed URL that represents a path is an absolute path.
-///
-/// [`Url::to_file_path`] has a different code path for Windows and other operating systems, this
-/// can cause URLs to parse perfectly fine on Windows, but fail to parse on Linux. This function
-/// tries to parse the URL as a path on all operating systems.
-pub fn url_to_path(url: &Url) -> Option<PathBuf> {
+fn url_to_path_inner<T: From<String>>(url: &Url) -> Option<T> {
     if url.scheme() != "file" {
         return None;
     }
@@ -77,7 +71,29 @@ pub fn url_to_path(url: &Url) -> Option<PathBuf> {
         }
     }
 
-    Some(PathBuf::from(path))
+    Some(T::from(path))
+}
+
+/// Tries to convert a `file://` based URL to a path.
+///
+/// We assume that any passed URL that represents a path is an absolute path.
+///
+/// [`Url::to_file_path`] has a different code path for Windows and other operating systems, this
+/// can cause URLs to parse perfectly fine on Windows, but fail to parse on Linux. This function
+/// tries to parse the URL as a path on all operating systems.
+pub fn url_to_path(url: &Url) -> Option<PathBuf> {
+    url_to_path_inner(url)
+}
+
+/// Tries to convert a `file://` based URL to a path.
+///
+/// We assume that any passed URL that represents a path is an absolute path.
+///
+/// [`Url::to_file_path`] has a different code path for Windows and other operating systems, this
+/// can cause URLs to parse perfectly fine on Windows, but fail to parse on Linux. This function
+/// tries to parse the URL as a path on all operating systems.
+pub fn url_to_typed_path(url: &Url) -> Option<Utf8TypedPathBuf> {
+    url_to_path_inner(url)
 }
 
 const FRAGMENT: &AsciiSet = &CONTROLS.add(b' ').add(b'"').add(b'<').add(b'>').add(b'`');

crates/rattler-bin/src/commands/create.rs

@@ -313,10 +313,10 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
 /// Prints the operations of the transaction to the console.
 fn print_transaction(transaction: &Transaction<PrefixRecord, RepoDataRecord>) {
     let format_record = |r: &RepoDataRecord| {
-        let direct_url_print = if r.clone().channel.is_empty() {
-            r.url.as_str()
+        let direct_url_print = if let Some(channel) = &r.channel {
+            channel.clone()
         } else {
-            ""
+            String::new()
         };
 
         format!(

crates/rattler/src/install/mod.rs

@@ -762,7 +762,7 @@ mod test {
         };
 
         test_install_python(
-            packages.filter_map(|p| Some(p.as_conda()?.url().clone())),
+            packages.filter_map(|p| p.as_conda()?.location().as_url().cloned()),
             "conda-lock",
             current_platform,
         )

crates/rattler/src/install/transaction.rs

@@ -219,6 +219,11 @@ fn is_python_record(record: &PackageRecord) -> bool {
 
 /// Returns true if the `from` and `to` describe the same package content
 fn describe_same_content(from: &PackageRecord, to: &PackageRecord) -> bool {
+    // If one hash is set and the other is not, the packages are different
+    if from.sha256.is_some() != to.sha256.is_some() || from.md5.is_some() != to.md5.is_some() {
+        return false;
+    }
+
     // If the hashes of the packages match we consider them to be equal
     if let (Some(a), Some(b)) = (from.sha256.as_ref(), to.sha256.as_ref()) {
         return a == b;

crates/rattler/src/lib.rs

@@ -78,6 +78,6 @@ pub(crate) fn get_repodata_record(package_path: impl AsRef<std::path::Path>) ->
             .unwrap()
             .to_string(),
         url: url::Url::from_file_path(package_path).unwrap(),
-        channel: "test".to_string(),
+        channel: Some(String::from("test")),
     }
 }

crates/rattler_conda_types/src/match_spec/mod.rs

@@ -676,7 +676,7 @@ mod tests {
             ),
             file_name: String::from("mamba-1.0-py37_0"),
             url: url::Url::parse("https://mamba.io/mamba-1.0-py37_0.conda").unwrap(),
-            channel: String::from("mamba"),
+            channel: Some(String::from("mamba")),
         };
         let package_record = repodata_record.clone().package_record;
 

crates/rattler_conda_types/src/repo_data/mod.rs

@@ -226,7 +226,6 @@ impl RepoData {
     /// given the source of the data.
     pub fn into_repo_data_records(self, channel: &Channel) -> Vec<RepoDataRecord> {
         let mut records = Vec::with_capacity(self.packages.len() + self.conda_packages.len());
-        let channel_name = channel.canonical_name();
         let base_url = self.base_url().map(ToOwned::to_owned);
 
         // Determine the base_url of the channel
@@ -241,7 +240,7 @@ impl RepoData {
                     base_url.as_deref(),
                     &filename,
                 ),
-                channel: channel_name.clone(),
+                channel: Some(channel.base_url.as_str().to_string()),
                 package_record,
                 file_name: filename,
             });

crates/rattler_conda_types/src/repo_data_record.rs

@@ -22,7 +22,8 @@ pub struct RepoDataRecord {
     /// String representation of the channel where the package comes from. This could be a URL but
     /// it could also be a channel name. Personally I would always add the complete URL here to be
     /// explicit about where the package came from.
-    pub channel: String,
+    /// TODO: Refactor this into `Source` which can be a "name", "channelurl", or "direct url".
+    pub channel: Option<String>,
 }
 
 impl AsRef<PackageRecord> for RepoDataRecord {

crates/rattler_lock/Cargo.toml

@@ -24,8 +24,10 @@ serde = { workspace = true, features = ["derive"] }
 serde_yaml = { workspace = true }
 serde_with = { workspace = true, features = ["indexmap_2"] }
 serde_repr = { workspace = true }
+serde-value = { workspace = true }
 thiserror = { workspace = true }
 url = { workspace = true, features = ["serde"] }
+typed-path = { workspace = true }
 
 [dev-dependencies]
 insta = { workspace = true, features = ["yaml"] }

crates/rattler_lock/src/builder.rs

@@ -11,11 +11,106 @@ use pep508_rs::ExtraName;
 use rattler_conda_types::Platform;
 
 use crate::{
-    file_format_version::FileFormatVersion, Channel, CondaPackageData, EnvironmentData,
-    EnvironmentPackageData, LockFile, LockFileInner, Package, PypiIndexes, PypiPackageData,
-    PypiPackageEnvironmentData,
+    file_format_version::FileFormatVersion, Channel, CondaBinaryData, CondaPackageData,
+    CondaSourceData, EnvironmentData, EnvironmentPackageData, LockFile, LockFileInner,
+    LockedPackageRef, PypiIndexes, PypiPackageData, PypiPackageEnvironmentData, UrlOrPath,
 };
 
+/// Information about a single locked package in an environment.
+#[derive(Debug, Clone)]
+#[allow(clippy::large_enum_variant)]
+pub enum LockedPackage {
+    /// A conda package
+    Conda(CondaPackageData),
+
+    /// A pypi package in an environment
+    Pypi(PypiPackageData, PypiPackageEnvironmentData),
+}
+
+impl From<LockedPackageRef<'_>> for LockedPackage {
+    fn from(value: LockedPackageRef<'_>) -> Self {
+        match value {
+            LockedPackageRef::Conda(data) => LockedPackage::Conda(data.clone()),
+            LockedPackageRef::Pypi(data, env) => LockedPackage::Pypi(data.clone(), env.clone()),
+        }
+    }
+}
+
+impl From<CondaPackageData> for LockedPackage {
+    fn from(value: CondaPackageData) -> Self {
+        LockedPackage::Conda(value)
+    }
+}
+
+impl From<(PypiPackageData, PypiPackageEnvironmentData)> for LockedPackage {
+    fn from((data, env): (PypiPackageData, PypiPackageEnvironmentData)) -> Self {
+        LockedPackage::Pypi(data, env)
+    }
+}
+
+impl LockedPackage {
+    /// Returns the name of the package as it occurs in the lock file. This
+    /// might not be the normalized name.
+    pub fn name(&self) -> &str {
+        match self {
+            LockedPackage::Conda(data) => data.record().name.as_source(),
+            LockedPackage::Pypi(data, _) => data.name.as_ref(),
+        }
+    }
+
+    /// Returns the location of the package.
+    pub fn location(&self) -> &UrlOrPath {
+        match self {
+            LockedPackage::Conda(data) => data.location(),
+            LockedPackage::Pypi(data, _) => &data.location,
+        }
+    }
+
+    /// Returns the conda package data if this is a conda package.
+    pub fn as_conda(&self) -> Option<&CondaPackageData> {
+        match self {
+            LockedPackage::Conda(data) => Some(data),
+            LockedPackage::Pypi(..) => None,
+        }
+    }
+
+    /// Returns the pypi package data if this is a pypi package.
+    pub fn as_pypi(&self) -> Option<(&PypiPackageData, &PypiPackageEnvironmentData)> {
+        match self {
+            LockedPackage::Conda(..) => None,
+            LockedPackage::Pypi(data, env) => Some((data, env)),
+        }
+    }
+
+    /// Returns the package as a binary conda package if this is a binary conda
+    /// package.
+    pub fn as_binary_conda(&self) -> Option<&CondaBinaryData> {
+        self.as_conda().and_then(CondaPackageData::as_binary)
+    }
+
+    /// Returns the package as a source conda package if this is a source conda
+    /// package.
+    pub fn as_source_conda(&self) -> Option<&CondaSourceData> {
+        self.as_conda().and_then(CondaPackageData::as_source)
+    }
+
+    /// Returns the conda package data if this is a conda package.
+    pub fn into_conda(self) -> Option<CondaPackageData> {
+        match self {
+            LockedPackage::Conda(data) => Some(data),
+            LockedPackage::Pypi(..) => None,
+        }
+    }
+
+    /// Returns the pypi package data if this is a pypi package.
+    pub fn into_pypi(self) -> Option<(PypiPackageData, PypiPackageEnvironmentData)> {
+        match self {
+            LockedPackage::Conda(..) => None,
+            LockedPackage::Pypi(data, env) => Some((data, env)),
+        }
+    }
+}
+
 /// A struct to incrementally build a lock-file.
 #[derive(Default)]
 pub struct LockFileBuilder {
@@ -162,7 +257,7 @@ impl LockFileBuilder {
         mut self,
         environment: impl Into<String>,
         platform: Platform,
-        locked_package: Package,
+        locked_package: LockedPackage,
     ) -> Self {
         self.add_package(environment, platform, locked_package);
         self
@@ -174,18 +269,13 @@ impl LockFileBuilder {
         &mut self,
         environment: impl Into<String>,
         platform: Platform,
-        locked_package: Package,
+        locked_package: LockedPackage,
     ) -> &mut Self {
         match locked_package {
-            Package::Conda(p) => {
-                self.add_conda_package(environment, platform, p.package_data().clone())
+            LockedPackage::Conda(p) => self.add_conda_package(environment, platform, p),
+            LockedPackage::Pypi(data, env_data) => {
+                self.add_pypi_package(environment, platform, data, env_data)
             }
-            Package::Pypi(p) => self.add_pypi_package(
-                environment,
-                platform,
-                p.package_data().clone(),
-                p.environment_data().clone(),
-            ),
         }
     }