From ed561f34c117bfb7dab912fc8bc2f47ed0f38d9c Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Thu, 23 May 2024 10:53:47 +0200 Subject: [PATCH] feat: removed Ord and more (#673) * Turns `purls` into a BtreeSet to ensure consistent ordering (fixes https://github.com/prefix-dev/pixi/issues/1367) * Removed `Ord` and `PartialOrd` from `PackageRecord`. It doesnt make sense. * Bumped `resolvo` to remove `Ord` constraint from `Version`. * Conversion from `CondaPackage` and `PypiPackage` to Package. * The ability to add packages to the lock file builder from another environment. --- Cargo.toml | 2 +- crates/rattler_conda_types/Cargo.toml | 4 +- .../rattler_conda_types/src/repo_data/mod.rs | 97 ++++---- .../src/repo_data/patches.rs | 3 +- .../src/repo_data_record.rs | 2 +- crates/rattler_lock/src/builder.rs | 73 ++++-- crates/rattler_lock/src/conda.rs | 1 - crates/rattler_lock/src/lib.rs | 219 +++++++++++------- crates/rattler_lock/src/parse/v3.rs | 2 +- .../src/utils/serde/raw_conda_package_data.rs | 3 +- crates/rattler_solve/src/resolvo/mod.rs | 29 ++- py-rattler/Cargo.lock | 35 +-- 12 files changed, 294 insertions(+), 176 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c3a168951..25a045e4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,7 +108,7 @@ regex = "1.10.4" reqwest = { version = "0.12.3", default-features = false } reqwest-middleware = "0.3.0" reqwest-retry = "0.5.0" -resolvo = { version = "0.4.0" } +resolvo = { version = "0.4.1" } retry-policies = { version = "0.3.0", default-features = false } rmp-serde = { version = "1.2.0" } rstest = { version = "0.19.0" } diff --git a/crates/rattler_conda_types/Cargo.toml b/crates/rattler_conda_types/Cargo.toml index 69df1be5d..716e1bd60 100644 --- a/crates/rattler_conda_types/Cargo.toml +++ b/crates/rattler_conda_types/Cargo.toml @@ -20,8 +20,8 @@ itertools = { workspace = true } lazy-regex = { workspace = true } nom = { workspace = true } purl = { workspace = true, features = ["serde"] } -rattler_digest = { path="../rattler_digest", version = "0.19.4", default-features = false, features = ["serde"] } -rattler_macros = { path="../rattler_macros", version = "0.19.3", default-features = false } +rattler_digest = { path = "../rattler_digest", version = "0.19.4", default-features = false, features = ["serde"] } +rattler_macros = { path = "../rattler_macros", version = "0.19.3", default-features = false } regex = { workspace = true } serde = { workspace = true, features = ["derive", "rc"] } serde_json = { workspace = true } diff --git a/crates/rattler_conda_types/src/repo_data/mod.rs b/crates/rattler_conda_types/src/repo_data/mod.rs index 9c4a8ca6c..273d91325 100644 --- a/crates/rattler_conda_types/src/repo_data/mod.rs +++ b/crates/rattler_conda_types/src/repo_data/mod.rs @@ -1,31 +1,32 @@ -//! Defines [`RepoData`]. `RepoData` stores information of all packages present in a subdirectory -//! of a channel. It provides indexing functionality. +//! Defines [`RepoData`]. `RepoData` stores information of all packages present +//! in a subdirectory of a channel. It provides indexing functionality. pub mod patches; pub mod sharded; mod topological_sort; -use std::borrow::Cow; -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt::{Display, Formatter}; -use std::path::Path; +use std::{ + borrow::Cow, + collections::{BTreeMap, BTreeSet}, + fmt::{Display, Formatter}, + path::Path, +}; use fxhash::{FxHashMap, FxHashSet}; - use rattler_digest::{serde::SerializableHash, Md5Hash, Sha256Hash}; +use rattler_macros::sorted; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, skip_serializing_none, OneOrMany}; use thiserror::Error; use url::Url; -use rattler_macros::sorted; - use crate::{ build_spec::BuildNumber, package::IndexJson, utils::serde::DeserializeFromStrUnchecked, Channel, NoArchType, PackageName, PackageUrl, Platform, RepoDataRecord, VersionWithSource, }; -/// [`RepoData`] is an index of package binaries available on in a subdirectory of a Conda channel. +/// [`RepoData`] is an index of package binaries available on in a subdirectory +/// of a Conda channel. // Note: we cannot use the sorted macro here, because the `packages` and `conda_packages` fields are // serialized in a special way. Therefore we do it manually. #[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Clone)] @@ -37,8 +38,9 @@ pub struct RepoData { #[serde(serialize_with = "sort_map_alphabetically")] pub packages: FxHashMap, - /// The conda packages contained in the repodata.json file (under a different key for - /// backwards compatibility with previous conda versions) + /// The conda packages contained in the repodata.json file (under a + /// different key for backwards compatibility with previous conda + /// versions) #[serde( default, rename = "packages.conda", @@ -46,7 +48,8 @@ pub struct RepoData { )] pub conda_packages: FxHashMap, - /// removed packages (files are still accessible, but they are not installable like regular packages) + /// removed packages (files are still accessible, but they are not + /// installable like regular packages) #[serde( default, serialize_with = "sort_set_alphabetically", @@ -70,12 +73,12 @@ pub struct ChannelInfo { pub base_url: Option, } -/// A single record in the Conda repodata. A single record refers to a single binary distribution -/// of a package on a Conda channel. +/// A single record in the Conda repodata. A single record refers to a single +/// binary distribution of a package on a Conda channel. #[serde_as] #[skip_serializing_none] #[sorted] -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Ord, PartialOrd, Clone, Hash)] +#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Clone, Hash)] pub struct PackageRecord { /// Optionally the architecture the package supports pub arch: Option, @@ -86,10 +89,11 @@ pub struct PackageRecord { /// The build number of the package pub build_number: BuildNumber, - /// Additional constraints on packages. `constrains` are different from `depends` in that packages - /// specified in `depends` must be installed next to this package, whereas packages specified in - /// `constrains` are not required to be installed, but if they are installed they must follow these - /// constraints. + /// Additional constraints on packages. `constrains` are different from + /// `depends` in that packages specified in `depends` must be installed + /// next to this package, whereas packages specified in `constrains` are + /// not required to be installed, but if they are installed they must follow + /// these constraints. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub constrains: Vec, @@ -97,8 +101,9 @@ pub struct PackageRecord { #[serde(default)] pub depends: Vec, - /// Features are a deprecated way to specify different feature sets for the conda solver. This is not - /// supported anymore and should not be used. Instead, `mutex` packages should be used to specify + /// Features are a deprecated way to specify different feature sets for the + /// conda solver. This is not supported anymore and should not be used. + /// Instead, `mutex` packages should be used to specify /// mutually exclusive features. pub features: Option, @@ -122,25 +127,25 @@ pub struct PackageRecord { #[serde_as(deserialize_as = "DeserializeFromStrUnchecked")] pub name: PackageName, - /// If this package is independent of architecture this field specifies in what way. See - /// [`NoArchType`] for more information. + /// If this package is independent of architecture this field specifies in + /// what way. See [`NoArchType`] for more information. #[serde(skip_serializing_if = "NoArchType::is_none")] pub noarch: NoArchType, /// Optionally the platform the package supports pub platform: Option, // Note that this does not match the [`Platform`] enum.. - /// Package identifiers of packages that are equivalent to this package but from other - /// ecosystems. + /// Package identifiers of packages that are equivalent to this package but + /// from other ecosystems. /// starting from 0.23.2, this field became [`Option>`]. /// This was done to support older lockfiles, /// where we didn't differentiate between empty purl and missing one. - /// Now, None:: means that the purl is missing, and it will be tried to filled in. - /// So later it can be one of the following: - /// [`Some(vec![])`] means that the purl is empty and package is not pypi one. - /// [`Some([`PackageUrl`])`] means that it is a pypi package. + /// Now, None:: means that the purl is missing, and it will be tried to + /// filled in. So later it can be one of the following: + /// [`Some(vec![])`] means that the purl is empty and package is not pypi + /// one. [`Some([`PackageUrl`])`] means that it is a pypi package. #[serde(default, skip_serializing_if = "Option::is_none")] - pub purls: Option>, + pub purls: Option>, /// Optionally a SHA256 hash of the package archive #[serde_as(as = "Option>")] @@ -157,8 +162,9 @@ pub struct PackageRecord { #[serde_as(as = "Option")] pub timestamp: Option>, - /// Track features are nowadays only used to downweight packages (ie. give them less priority). To - /// that effect, the number of track features is counted (number of commas) and the package is downweighted + /// Track features are nowadays only used to downweight packages (ie. give + /// them less priority). To that effect, the number of track features is + /// counted (number of commas) and the package is downweighted /// by the number of track_features. #[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde_as(as = "OneOrMany<_>")] @@ -201,8 +207,8 @@ impl RepoData { self.info.as_ref().and_then(|i| i.base_url.as_deref()) } - /// Builds a [`Vec`] from the packages in a [`RepoData`] given the source of the - /// data. + /// Builds a [`Vec`] from the packages in a [`RepoData`] + /// given the source of the data. pub fn into_repo_data_records(self, channel: &Channel) -> Vec { let mut records = Vec::with_capacity(self.packages.len() + self.conda_packages.len()); let channel_name = channel.canonical_name(); @@ -273,7 +279,8 @@ fn add_trailing_slash(url: &Url) -> Cow<'_, Url> { } impl PackageRecord { - /// A simple helper method that constructs a `PackageRecord` with the bare minimum values. + /// A simple helper method that constructs a `PackageRecord` with the bare + /// minimum values. pub fn new(name: PackageName, version: impl Into, build: String) -> Self { Self { arch: None, @@ -302,8 +309,9 @@ impl PackageRecord { /// Sorts the records topologically. /// - /// This function is deterministic, meaning that it will return the same result regardless of - /// the order of `records` and of the `depends` vector inside the records. + /// This function is deterministic, meaning that it will return the same + /// result regardless of the order of `records` and of the `depends` + /// vector inside the records. /// /// Note that this function only works for packages with unique names. pub fn sort_topologically + Clone>(records: Vec) -> Vec { @@ -338,8 +346,8 @@ pub enum ConvertSubdirError { /// # Why can we not use `Platform::FromStr`? /// /// We cannot use the [`Platform`] `FromStr` directly because `x86` and `x86_64` -/// are different architecture strings. Also some combinations have been removed, -/// because they have not been found. +/// are different architecture strings. Also some combinations have been +/// removed, because they have not been found. fn determine_subdir( platform: Option, arch: Option, @@ -377,7 +385,8 @@ fn determine_subdir( } impl PackageRecord { - /// Builds a [`PackageRecord`] from a [`IndexJson`] and optionally a size, sha256 and md5 hash. + /// Builds a [`PackageRecord`] from a [`IndexJson`] and optionally a size, + /// sha256 and md5 hash. pub fn from_index_json( index: IndexJson, size: Option, @@ -435,10 +444,12 @@ fn sort_set_alphabetically( #[cfg(test)] mod test { - use crate::repo_data::{compute_package_url, determine_subdir}; use fxhash::FxHashMap; - use crate::{Channel, ChannelConfig, RepoData}; + use crate::{ + repo_data::{compute_package_url, determine_subdir}, + Channel, ChannelConfig, RepoData, + }; // isl-0.12.2-1.tar.bz2 // gmp-5.1.2-6.tar.bz2 diff --git a/crates/rattler_conda_types/src/repo_data/patches.rs b/crates/rattler_conda_types/src/repo_data/patches.rs index 5db6708f4..5c8f75184 100644 --- a/crates/rattler_conda_types/src/repo_data/patches.rs +++ b/crates/rattler_conda_types/src/repo_data/patches.rs @@ -3,6 +3,7 @@ use fxhash::{FxHashMap, FxHashSet}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, skip_serializing_none, OneOrMany}; +use std::collections::BTreeSet; use std::io; use std::path::Path; @@ -97,7 +98,7 @@ pub struct PackageRecordPatch { /// Package identifiers of packages that are equivalent to this package but from other /// ecosystems. - pub purls: Option>, + pub purls: Option>, } /// Repodata patch instructions for a single subdirectory. See [`RepoDataPatch`] for more diff --git a/crates/rattler_conda_types/src/repo_data_record.rs b/crates/rattler_conda_types/src/repo_data_record.rs index b05e06f61..819a533d6 100644 --- a/crates/rattler_conda_types/src/repo_data_record.rs +++ b/crates/rattler_conda_types/src/repo_data_record.rs @@ -6,7 +6,7 @@ use url::Url; /// Information about a package from repodata. It includes a [`crate::PackageRecord`] but it also stores /// the source of the data (like the url and the channel). -#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Ord, PartialOrd, Clone, Hash)] +#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Clone, Hash)] pub struct RepoDataRecord { /// The data stored in the repodata.json. #[serde(flatten)] diff --git a/crates/rattler_lock/src/builder.rs b/crates/rattler_lock/src/builder.rs index 2a24872bd..152c4dc7d 100644 --- a/crates/rattler_lock/src/builder.rs +++ b/crates/rattler_lock/src/builder.rs @@ -1,17 +1,19 @@ //! Builder for the creation of lock files. -use crate::file_format_version::FileFormatVersion; -use crate::{ - Channel, CondaPackageData, EnvironmentData, EnvironmentPackageData, LockFile, LockFileInner, - PypiIndexes, PypiPackageData, PypiPackageEnvironmentData, +use std::{ + collections::{BTreeSet, HashMap}, + sync::Arc, }; + use fxhash::FxHashMap; use indexmap::{IndexMap, IndexSet}; use pep508_rs::ExtraName; use rattler_conda_types::Platform; -use std::{ - collections::{BTreeSet, HashMap}, - sync::Arc, + +use crate::{ + file_format_version::FileFormatVersion, Channel, CondaPackageData, EnvironmentData, + EnvironmentPackageData, LockFile, LockFileInner, Package, PypiIndexes, PypiPackageData, + PypiPackageEnvironmentData, }; /// A struct to incrementally build a lock-file. @@ -68,9 +70,9 @@ impl LockFileBuilder { /// Adds a conda locked package to a specific environment and platform. /// - /// This function is similar to [`Self::with_conda_package`] but differs in that it takes a - /// mutable reference to self instead of consuming it. This allows for a more fluent with - /// chaining calls. + /// This function is similar to [`Self::with_conda_package`] but differs in + /// that it takes a mutable reference to self instead of consuming it. + /// This allows for a more fluent with chaining calls. pub fn add_conda_package( &mut self, environment: impl Into, @@ -102,9 +104,9 @@ impl LockFileBuilder { /// Adds a pypi locked package to a specific environment and platform. /// - /// This function is similar to [`Self::with_pypi_package`] but differs in that it takes a - /// mutable reference to self instead of consuming it. This allows for a more fluent with - /// chaining calls. + /// This function is similar to [`Self::with_pypi_package`] but differs in + /// that it takes a mutable reference to self instead of consuming it. + /// This allows for a more fluent with chaining calls. pub fn add_pypi_package( &mut self, environment: impl Into, @@ -141,9 +143,9 @@ impl LockFileBuilder { /// Adds a conda locked package to a specific environment and platform. /// - /// This function is similar to [`Self::add_conda_package`] but differs in that it consumes - /// `self` instead of taking a mutable reference. This allows for a better interface when - /// modifying an existing instance. + /// This function is similar to [`Self::add_conda_package`] but differs in + /// that it consumes `self` instead of taking a mutable reference. This + /// allows for a better interface when modifying an existing instance. pub fn with_conda_package( mut self, environment: impl Into, @@ -154,11 +156,44 @@ impl LockFileBuilder { self } + /// Adds a package from another environment to a specific environment and + /// platform. + pub fn with_package( + mut self, + environment: impl Into, + platform: Platform, + locked_package: Package, + ) -> Self { + self.add_package(environment, platform, locked_package); + self + } + + /// Adds a package from another environment to a specific environment and + /// platform. + pub fn add_package( + &mut self, + environment: impl Into, + platform: Platform, + locked_package: Package, + ) -> &mut Self { + match locked_package { + Package::Conda(p) => { + self.add_conda_package(environment, platform, p.package_data().clone()) + } + Package::Pypi(p) => self.add_pypi_package( + environment, + platform, + p.package_data().clone(), + p.environment_data().clone(), + ), + } + } + /// Adds a pypi locked package to a specific environment and platform. /// - /// This function is similar to [`Self::add_pypi_package`] but differs in that it consumes - /// `self` instead of taking a mutable reference. This allows for a better interface when - /// modifying an existing instance. + /// This function is similar to [`Self::add_pypi_package`] but differs in + /// that it consumes `self` instead of taking a mutable reference. This + /// allows for a better interface when modifying an existing instance. pub fn with_pypi_package( mut self, environment: impl Into, diff --git a/crates/rattler_lock/src/conda.rs b/crates/rattler_lock/src/conda.rs index 662e48647..353da9aad 100644 --- a/crates/rattler_lock/src/conda.rs +++ b/crates/rattler_lock/src/conda.rs @@ -2,7 +2,6 @@ use rattler_conda_types::{PackageRecord, RepoDataRecord}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, skip_serializing_none}; use std::cmp::Ordering; -use std::hash::Hash; use url::Url; /// A locked conda dependency is just a [`PackageRecord`] with some additional information on where diff --git a/crates/rattler_lock/src/lib.rs b/crates/rattler_lock/src/lib.rs index fd086618a..0fbcdadaa 100644 --- a/crates/rattler_lock/src/lib.rs +++ b/crates/rattler_lock/src/lib.rs @@ -1,78 +1,93 @@ #![deny(missing_docs, dead_code)] -//! Definitions for a lock-file format that stores information about pinned dependencies from both -//! the Conda and Pypi ecosystem. +//! Definitions for a lock-file format that stores information about pinned +//! dependencies from both the Conda and Pypi ecosystem. //! //! The crate is structured in two API levels. //! -//! 1. The top level API accessible through the [`LockFile`] type that exposes high level access to -//! the lock-file. This API is intended to be relatively stable and is the preferred way to -//! interact with the lock-file. -//! 2. The `*Data` types. These are lower level types that expose more of the internal data -//! structures used in the crate. These types are not intended to be stable and are subject to -//! change over time. These types are used internally by the top level API. Also note that only -//! a subset of the `*Data` types are exposed. See `[crate::PyPiPackageData]`, +//! 1. The top level API accessible through the [`LockFile`] type that exposes +//! high level access to the lock-file. This API is intended to be relatively +//! stable and is the preferred way to interact with the lock-file. +//! 2. The `*Data` types. These are lower level types that expose more of the +//! internal data structures used in the crate. These types are not intended +//! to be stable and are subject to change over time. These types are used +//! internally by the top level API. Also note that only a subset of the +//! `*Data` types are exposed. See `[crate::PyPiPackageData]`, //! `[crate::CondaPackageData]` for examples. //! //! ## Design goals //! //! The goal of the lock-file format is: //! -//! * To be complete. The lock-file should contain all the information needed to recreate -//! environments even years after it was created. As long as the package data persists that a -//! lock-file refers to, it should be possible to recreate the environment. -//! * To be human readable. Although lock-files are not intended to be edited by hand, they should -//! be relatively easy to read and understand. So that when a lock-file is checked into version -//! control and someone looks at the diff, they can understand what changed. -//! * To be easily parsable. It should be fairly straightforward to create a parser for the format -//! so that it can be used in other tools. -//! * To reduce diff size when the content changes. The order of content in the serialized lock-file -//! should be fixed to ensure that the diff size is minimized when the content changes. -//! * To be reproducible. Recreating the lock-file with the exact same input (including externally -//! fetched data) should yield the same lock-file byte-for-byte. -//! * To be statically verifiable. Given the specifications of the packages that went into a -//! lock-file it should be possible to cheaply verify whether or not the specifications are still -//! satisfied by the packages stored in the lock-file. -//! * Backward compatible. Older version of lock-files should still be readable by never versions of -//! this crate. +//! * To be complete. The lock-file should contain all the information needed to +//! recreate environments even years after it was created. As long as the +//! package data persists that a lock-file refers to, it should be possible to +//! recreate the environment. +//! * To be human readable. Although lock-files are not intended to be edited by +//! hand, they should be relatively easy to read and understand. So that when +//! a lock-file is checked into version control and someone looks at the diff, +//! they can understand what changed. +//! * To be easily parsable. It should be fairly straightforward to create a +//! parser for the format so that it can be used in other tools. +//! * To reduce diff size when the content changes. The order of content in the +//! serialized lock-file should be fixed to ensure that the diff size is +//! minimized when the content changes. +//! * To be reproducible. Recreating the lock-file with the exact same input +//! (including externally fetched data) should yield the same lock-file +//! byte-for-byte. +//! * To be statically verifiable. Given the specifications of the packages that +//! went into a lock-file it should be possible to cheaply verify whether or +//! not the specifications are still satisfied by the packages stored in the +//! lock-file. +//! * Backward compatible. Older version of lock-files should still be readable +//! by never versions of this crate. //! //! ## Relation to conda-lock //! //! Initially the lock-file format was based on [`conda-lock`](https://github.com/conda/conda-lock) -//! but over time significant changes have been made compared to the original conda-lock format. -//! Conda-lock files (e.g. `conda-lock.yml` files) can still be parsed by this crate but the -//! serialization format changed significantly. This means files created by this crate are not -//! compatible with conda-lock. +//! but over time significant changes have been made compared to the original +//! conda-lock format. Conda-lock files (e.g. `conda-lock.yml` files) can still +//! be parsed by this crate but the serialization format changed significantly. +//! This means files created by this crate are not compatible with conda-lock. //! -//! Conda-lock stores a lot of metadata to be able to verify if the lock-file is still valid given -//! the sources/inputs. For example conda-lock contains a `content-hash` which is a hash of all the -//! input data of the lock-file. -//! This crate approaches this differently by storing enough information in the lock-file to be able -//! to verify if the lock-file still satisfies an input/source without requiring additional input -//! (e.g. network requests) or expensive solves. We call this static satisfiability verification. +//! Conda-lock stores a lot of metadata to be able to verify if the lock-file is +//! still valid given the sources/inputs. For example conda-lock contains a +//! `content-hash` which is a hash of all the input data of the lock-file. +//! This crate approaches this differently by storing enough information in the +//! lock-file to be able to verify if the lock-file still satisfies an +//! input/source without requiring additional input (e.g. network requests) or +//! expensive solves. We call this static satisfiability verification. //! -//! Conda-lock stores a custom __partial__ representation of a [`rattler_conda_types::RepoDataRecord`] -//! in the lock-file. This poses a problem when incrementally updating an environment. To only -//! partially update packages in the lock-file without completely recreating it, the records stored -//! in the lock-file need to be passed to the solver as "preferred" packages. Since -//! [`rattler_conda_types::MatchSpec`] can match on any field present in a -//! [`rattler_conda_types::PackageRecord`] we need to store all fields in the lock-file not just a -//! subset. -//! To that end this crate stores the full [`rattler_conda_types::PackageRecord`] in the lock-file. -//! This allows completely recreating the record that was read from repodata when the lock-file was -//! created which will allow a correct incremental update. +//! Conda-lock stores a custom __partial__ representation of a +//! [`rattler_conda_types::RepoDataRecord`] in the lock-file. This poses a +//! problem when incrementally updating an environment. To only partially update +//! packages in the lock-file without completely recreating it, the records +//! stored in the lock-file need to be passed to the solver as "preferred" +//! packages. Since [`rattler_conda_types::MatchSpec`] can match on any field +//! present in a [`rattler_conda_types::PackageRecord`] we need to store all +//! fields in the lock-file not just a subset. +//! To that end this crate stores the full +//! [`rattler_conda_types::PackageRecord`] in the lock-file. This allows +//! completely recreating the record that was read from repodata when the +//! lock-file was created which will allow a correct incremental update. //! -//! Conda-lock requires users to create multiple lock-files when they want to store multiple -//! environments. This crate allows storing multiple environments for different platforms and with -//! different channels in a single lock-file. This allows storing production- and test environments -//! in a single file. +//! Conda-lock requires users to create multiple lock-files when they want to +//! store multiple environments. This crate allows storing multiple environments +//! for different platforms and with different channels in a single lock-file. +//! This allows storing production- and test environments in a single file. + +use std::{ + borrow::Cow, + collections::{BTreeSet, HashMap}, + io::Read, + path::Path, + str::FromStr, + sync::Arc, +}; use fxhash::FxHashMap; use pep508_rs::{ExtraName, Requirement}; use rattler_conda_types::{MatchSpec, PackageRecord, Platform, RepoDataRecord}; -use std::collections::{BTreeSet, HashMap}; -use std::sync::Arc; -use std::{borrow::Cow, io::Read, path::Path, str::FromStr}; use url::Url; mod builder; @@ -96,16 +111,19 @@ pub use pypi::{PypiPackageData, PypiPackageEnvironmentData, PypiSourceTreeHashab pub use pypi_indexes::{FindLinksUrlOrPath, PypiIndexes}; pub use url_or_path::UrlOrPath; -/// The name of the default environment in a [`LockFile`]. This is the environment name that is used -/// when no explicit environment name is specified. +/// The name of the default environment in a [`LockFile`]. This is the +/// environment name that is used when no explicit environment name is +/// specified. pub const DEFAULT_ENVIRONMENT_NAME: &str = "default"; /// Represents a lock-file for both Conda packages and Pypi packages. /// -/// Lock-files can store information for multiple platforms and for multiple environments. +/// Lock-files can store information for multiple platforms and for multiple +/// environments. /// -/// The high-level API provided by this type holds internal references to the data. Its is therefore -/// cheap to clone this type and any type derived from it (e.g. [`Environment`] or [`Package`]). +/// The high-level API provided by this type holds internal references to the +/// data. Its is therefore cheap to clone this type and any type derived from it +/// (e.g. [`Environment`] or [`Package`]). #[derive(Clone, Default)] pub struct LockFile { inner: Arc, @@ -123,9 +141,10 @@ struct LockFileInner { environment_lookup: FxHashMap, } -/// An package used in an environment. Selects a type of package based on the enum and might contain -/// additional data that is specific to the environment. For instance different environments might -/// select the same Pypi package but with different extras. +/// An package used in an environment. Selects a type of package based on the +/// enum and might contain additional data that is specific to the environment. +/// For instance different environments might select the same Pypi package but +/// with different extras. #[derive(Clone, Copy, Debug)] enum EnvironmentPackageData { Conda(usize), @@ -134,8 +153,8 @@ enum EnvironmentPackageData { /// Information about a specific environment in the lock file. /// -/// This only needs to store information about an environment that cannot be derived from the -/// packages itself. +/// This only needs to store information about an environment that cannot be +/// derived from the packages itself. /// /// The default environment is called "default". #[derive(Clone, Debug)] @@ -146,14 +165,14 @@ struct EnvironmentData { /// The pypi indexes used to solve the environment. indexes: Option, - /// For each individual platform this environment supports we store the package identifiers - /// associated with the environment. + /// For each individual platform this environment supports we store the + /// package identifiers associated with the environment. packages: FxHashMap>, } impl LockFile { - /// Constructs a new lock-file builder. This is the preferred way to constructs a lock-file - /// programmatically. + /// Constructs a new lock-file builder. This is the preferred way to + /// constructs a lock-file programmatically. pub fn builder() -> LockFileBuilder { LockFileBuilder::new() } @@ -187,7 +206,8 @@ impl LockFile { }) } - /// Returns the environment with the default name as defined by [`DEFAULT_ENVIRONMENT_NAME`]. + /// Returns the environment with the default name as defined by + /// [`DEFAULT_ENVIRONMENT_NAME`]. pub fn default_environment(&self) -> Option { self.environment(DEFAULT_ENVIRONMENT_NAME) } @@ -236,8 +256,8 @@ impl Environment { /// Returns the channels that are used by this environment. /// - /// Note that the order of the channels is significant. The first channel is the highest - /// priority channel. + /// Note that the order of the channels is significant. The first channel is + /// the highest priority channel. pub fn channels(&self) -> &[Channel] { &self.data().channels } @@ -264,7 +284,8 @@ impl Environment { ) } - /// Returns an iterator over all packages and platforms defined for this environment + /// Returns an iterator over all packages and platforms defined for this + /// environment pub fn packages_by_platform( &self, ) -> impl Iterator< @@ -309,7 +330,8 @@ impl Environment { .collect() } - /// Returns all conda packages for all platforms and converts them to [`RepoDataRecord`]. + /// Returns all conda packages for all platforms and converts them to + /// [`RepoDataRecord`]. pub fn conda_repodata_records( &self, ) -> Result>, ConversionError> { @@ -332,9 +354,10 @@ impl Environment { .collect() } - /// Takes all the conda packages, converts them to [`RepoDataRecord`] and returns them or - /// returns an error if the conversion failed. Returns `None` if the specified platform is not - /// defined for this environment. + /// Takes all the conda packages, converts them to [`RepoDataRecord`] and + /// returns them or returns an error if the conversion failed. Returns + /// `None` if the specified platform is not defined for this + /// environment. pub fn conda_repodata_records_for_platform( &self, platform: Platform, @@ -355,8 +378,9 @@ impl Environment { .map(Some) } - /// Returns all the pypi packages and their associated environment data for the specified - /// platform. Returns `None` if the platform is not defined for this environment. + /// Returns all the pypi packages and their associated environment data for + /// the specified platform. Returns `None` if the platform is not + /// defined for this environment. pub fn pypi_packages_for_platform( &self, platform: Platform, @@ -395,9 +419,21 @@ pub enum Package { Pypi(PypiPackage), } +impl From for Package { + fn from(value: CondaPackage) -> Self { + Package::Conda(value) + } +} + +impl From for Package { + fn from(value: PypiPackage) -> Self { + Package::Pypi(value) + } +} + impl Package { - /// Constructs a new instance from a [`EnvironmentPackageData`] and a reference to the internal - /// data structure. + /// Constructs a new instance from a [`EnvironmentPackageData`] and a + /// reference to the internal data structure. fn from_env_package(data: EnvironmentPackageData, inner: Arc) -> Self { match data { EnvironmentPackageData::Conda(idx) => { @@ -421,8 +457,8 @@ impl Package { matches!(self, Self::Pypi(_)) } - /// Returns this instance as a [`CondaPackage`] if this instance represents a conda - /// package. + /// Returns this instance as a [`CondaPackage`] if this instance represents + /// a conda package. pub fn as_conda(&self) -> Option<&CondaPackage> { match self { Self::Conda(value) => Some(value), @@ -430,8 +466,8 @@ impl Package { } } - /// Returns this instance as a [`PypiPackage`] if this instance represents a pypi - /// package. + /// Returns this instance as a [`PypiPackage`] if this instance represents a + /// pypi package. pub fn as_pypi(&self) -> Option<&PypiPackage> { match self { Self::Conda(_) => None, @@ -439,8 +475,8 @@ impl Package { } } - /// Returns this instance as a [`CondaPackage`] if this instance represents a conda - /// package. + /// Returns this instance as a [`CondaPackage`] if this instance represents + /// a conda package. pub fn into_conda(self) -> Option { match self { Self::Conda(value) => Some(value), @@ -448,8 +484,8 @@ impl Package { } } - /// Returns this instance as a [`PypiPackage`] if this instance represents a pypi - /// package. + /// Returns this instance as a [`PypiPackage`] if this instance represents a + /// pypi package. pub fn into_pypi(self) -> Option { match self { Self::Conda(_) => None, @@ -600,16 +636,19 @@ pub struct PypiPackageDataRef<'p> { /// The package data. This information is deduplicated between environments. pub package: &'p PypiPackageData, - /// Environment specific data for the package. This information is specific to the environment. + /// Environment specific data for the package. This information is specific + /// to the environment. pub environment: &'p PypiPackageEnvironmentData, } #[cfg(test)] mod test { - use super::{LockFile, DEFAULT_ENVIRONMENT_NAME}; + use std::path::Path; + use rattler_conda_types::Platform; use rstest::*; - use std::path::Path; + + use super::{LockFile, DEFAULT_ENVIRONMENT_NAME}; #[rstest] #[case("v0/numpy-conda-lock.yml")] diff --git a/crates/rattler_lock/src/parse/v3.rs b/crates/rattler_lock/src/parse/v3.rs index 991ff305d..133426950 100644 --- a/crates/rattler_lock/src/parse/v3.rs +++ b/crates/rattler_lock/src/parse/v3.rs @@ -115,7 +115,7 @@ pub struct CondaLockedPackageV3 { #[serde_as(as = "Option")] pub timestamp: Option>, #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub purls: Vec, + pub purls: BTreeSet, } /// A function that enables parsing of lock files version 3 or lower. diff --git a/crates/rattler_lock/src/utils/serde/raw_conda_package_data.rs b/crates/rattler_lock/src/utils/serde/raw_conda_package_data.rs index 949c8a588..2390e7985 100644 --- a/crates/rattler_lock/src/utils/serde/raw_conda_package_data.rs +++ b/crates/rattler_lock/src/utils/serde/raw_conda_package_data.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_with::serde_as; use std::borrow::Cow; use std::cmp::Ordering; +use std::collections::BTreeSet; use url::Url; fn is_default(value: &T) -> bool { @@ -86,7 +87,7 @@ pub(crate) struct RawCondaPackageData<'a> { #[serde(default, skip_serializing_if = "Option::is_none")] pub license_family: Cow<'a, Option>, #[serde(default, skip_serializing_if = "Option::is_none")] - pub purls: Cow<'a, Option>>, + pub purls: Cow<'a, Option>>, #[serde(default, skip_serializing_if = "Option::is_none")] pub size: Cow<'a, Option>, diff --git a/crates/rattler_solve/src/resolvo/mod.rs b/crates/rattler_solve/src/resolvo/mod.rs index 809cf3106..8aa6411cc 100644 --- a/crates/rattler_solve/src/resolvo/mod.rs +++ b/crates/rattler_solve/src/resolvo/mod.rs @@ -13,8 +13,8 @@ use std::{ use chrono::{DateTime, Utc}; use itertools::Itertools; use rattler_conda_types::{ - package::ArchiveType, GenericVirtualPackage, MatchSpec, NamelessMatchSpec, PackageRecord, - ParseMatchSpecError, ParseStrictness, RepoDataRecord, + package::ArchiveType, GenericVirtualPackage, MatchSpec, NamelessMatchSpec, PackageName, + PackageRecord, ParseMatchSpecError, ParseStrictness, RepoDataRecord, }; use resolvo::{ Candidates, Dependencies, DependencyProvider, KnownDependencies, NameId, Pool, SolvableDisplay, @@ -108,13 +108,36 @@ impl<'a> VersionSet for SolverMatchSpec<'a> { } /// Wrapper around [`PackageRecord`] so that we can use it in resolvo pool -#[derive(Ord, PartialOrd, Eq, PartialEq)] +#[derive(Eq, PartialEq)] enum SolverPackageRecord<'a> { Record(&'a RepoDataRecord), VirtualPackage(&'a GenericVirtualPackage), } +impl<'a> PartialOrd for SolverPackageRecord<'a> { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl<'a> Ord for SolverPackageRecord<'a> { + fn cmp(&self, other: &Self) -> Ordering { + self.name() + .cmp(other.name()) + .then_with(|| self.version().cmp(other.version())) + .then_with(|| self.build_number().cmp(&other.build_number())) + .then_with(|| self.timestamp().cmp(&other.timestamp())) + } +} + impl<'a> SolverPackageRecord<'a> { + fn name(&self) -> &PackageName { + match self { + SolverPackageRecord::Record(rec) => &rec.package_record.name, + SolverPackageRecord::VirtualPackage(rec) => &rec.name, + } + } + fn version(&self) -> &rattler_conda_types::Version { match self { SolverPackageRecord::Record(rec) => rec.package_record.version.version(), diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index 2a14edaca..9704c02aa 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -72,7 +72,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9db67cd9cf4f56a10d2cbae6a3b552e5bd36701fd37b74a18c14a231bdf446c7" dependencies = [ "cfg-if", - "itertools", + "itertools 0.12.1", "libc", "serde", "serde_json", @@ -784,7 +784,7 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" name = "file_url" version = "0.1.1" dependencies = [ - "itertools", + "itertools 0.12.1", "percent-encoding", "typed-path", "url", @@ -1513,6 +1513,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -2001,7 +2010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b645dcde5f119c2c454a92d0dfa271a2a3b205da92e4292a68ead4bdbfde1f33" dependencies = [ "heck", - "itertools", + "itertools 0.12.1", "proc-macro2", "proc-macro2-diagnostics", "quote", @@ -2486,7 +2495,7 @@ dependencies = [ "futures", "fxhash", "indexmap 2.2.6", - "itertools", + "itertools 0.12.1", "memchr", "memmap2", "once_cell", @@ -2518,7 +2527,7 @@ dependencies = [ "fxhash", "glob", "hex", - "itertools", + "itertools 0.12.1", "lazy-regex", "nom", "purl", @@ -2573,7 +2582,7 @@ dependencies = [ "file_url", "fxhash", "indexmap 2.2.6", - "itertools", + "itertools 0.12.1", "pep440_rs", "pep508_rs", "purl", @@ -2609,7 +2618,7 @@ dependencies = [ "getrandom", "google-cloud-auth", "http 1.1.0", - "itertools", + "itertools 0.12.1", "keyring", "netrc-rs", "reqwest 0.12.4", @@ -2666,7 +2675,7 @@ dependencies = [ "http-cache-semantics", "humansize", "humantime", - "itertools", + "itertools 0.12.1", "json-patch", "libc", "md-5", @@ -2701,7 +2710,7 @@ version = "0.20.4" dependencies = [ "enum_dispatch", "indexmap 2.2.6", - "itertools", + "itertools 0.12.1", "rattler_conda_types", "serde_json", "shlex", @@ -2716,7 +2725,7 @@ version = "0.22.0" dependencies = [ "chrono", "futures", - "itertools", + "itertools 0.12.1", "rattler_conda_types", "rattler_digest", "resolvo", @@ -2920,15 +2929,15 @@ dependencies = [ [[package]] name = "resolvo" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2016584c3fd9df0fd859a7dcbc7fafdc7fdd2d87b53a576e8e63e62fad140e33" +checksum = "d299d168910c5d71f3c0f5441abe38ca4a6ae21f70fae909bfc6bead28f6620f" dependencies = [ "bitvec", "elsa", "event-listener 5.3.0", "futures", - "itertools", + "itertools 0.13.0", "petgraph", "tracing", ]