From 5c9f1e810b697f7c6152947f2bc14a927bb9d7cf Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Mon, 21 Aug 2023 18:44:35 +0200 Subject: [PATCH 1/8] fix: local version parsing in constraint (#281) --- crates/rattler_conda_types/src/version_spec/parse.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/rattler_conda_types/src/version_spec/parse.rs b/crates/rattler_conda_types/src/version_spec/parse.rs index f9cf13228..a3a68dc76 100644 --- a/crates/rattler_conda_types/src/version_spec/parse.rs +++ b/crates/rattler_conda_types/src/version_spec/parse.rs @@ -119,7 +119,7 @@ fn logical_constraint_parser(input: &str) -> IResult<&str, Constraint, ParseCons // Take everything that looks like a version and use that to parse the version. Any error means // no characters were detected that belong to the version. let (rest, version_str) = take_while1::<_, _, (&str, ErrorKind)>(|c: char| { - c.is_alphanumeric() || "!-_.*".contains(c) + c.is_alphanumeric() || "!-_.*+".contains(c) })(input) .map_err(|_| { nom::Err::Error(ParseConstraintError::InvalidVersion(ParseVersionError { @@ -188,7 +188,7 @@ pub(crate) fn constraint_parser(input: &str) -> IResult<&str, Constraint, ParseC #[cfg(test)] mod test { use super::*; - use crate::Version; + use crate::{Version, VersionSpec}; use std::str::FromStr; #[test] @@ -340,4 +340,9 @@ mod test { assert_eq!(constraint_parser("*"), Ok(("", Constraint::Any))); assert_eq!(constraint_parser("*.*"), Ok(("", Constraint::Any))); } + + #[test] + fn pixi_issue_278() { + assert!(VersionSpec::from_str("1.8.1+g6b29558").is_ok()); + } } From c4622533573a229ba11c81a8a4890a8e2767a204 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 22 Aug 2023 09:20:35 +0200 Subject: [PATCH 2/8] fix: add retry behavior for package cache downloads (#280) * fix: add retry for package cache * feat: add retry policy crate * test: add a test to verify the retry behavior --- crates/rattler-bin/src/commands/create.rs | 7 +- crates/rattler/Cargo.toml | 5 + crates/rattler/src/package_cache.rs | 188 +++++++++++++++++- crates/rattler_networking/Cargo.toml | 1 + crates/rattler_networking/src/lib.rs | 1 + .../rattler_networking/src/retry_policies.rs | 23 +++ 6 files changed, 215 insertions(+), 10 deletions(-) create mode 100644 crates/rattler_networking/src/retry_policies.rs diff --git a/crates/rattler-bin/src/commands/create.rs b/crates/rattler-bin/src/commands/create.rs index 51624775f..5b40a7c4f 100644 --- a/crates/rattler-bin/src/commands/create.rs +++ b/crates/rattler-bin/src/commands/create.rs @@ -11,7 +11,9 @@ use rattler_conda_types::{ Channel, ChannelConfig, GenericVirtualPackage, MatchSpec, PackageRecord, Platform, PrefixRecord, RepoDataRecord, Version, }; -use rattler_networking::{AuthenticatedClient, AuthenticationStorage}; +use rattler_networking::{ + retry_policies::default_retry_policy, AuthenticatedClient, AuthenticationStorage, +}; use rattler_repodata_gateway::fetch::{ CacheResult, DownloadProgress, FetchRepoDataError, FetchRepoDataOptions, }; @@ -397,10 +399,11 @@ async fn execute_operation( async { // Make sure the package is available in the package cache. let result = package_cache - .get_or_fetch_from_url( + .get_or_fetch_from_url_with_retry( &install_record.package_record, install_record.url.clone(), download_client.clone(), + default_retry_policy(), ) .map_ok(|cache_dir| Some((install_record.clone(), cache_dir))) .map_err(anyhow::Error::from) diff --git a/crates/rattler/Cargo.toml b/crates/rattler/Cargo.toml index 048314721..50c9c346c 100644 --- a/crates/rattler/Cargo.toml +++ b/crates/rattler/Cargo.toml @@ -56,3 +56,8 @@ rand = "0.8.5" rstest = "0.18.1" tracing-test = { version = "0.2.4" } insta = { version = "1.30.0", features = ["yaml"] } + +tokio = { version = "1.29.1", features = ["macros", "rt-multi-thread"] } +axum = "0.6.18" +tower-http = { version = "0.4.1", features = ["fs"] } +tower = { version = "0.4.13", default-features = false, features = ["util"] } diff --git a/crates/rattler/src/package_cache.rs b/crates/rattler/src/package_cache.rs index b611900b0..bad1f9aff 100644 --- a/crates/rattler/src/package_cache.rs +++ b/crates/rattler/src/package_cache.rs @@ -1,11 +1,16 @@ //! This module provides functionality to cache extracted Conda packages. See [`PackageCache`]. use crate::validation::validate_package_directory; +use chrono::Utc; use fxhash::FxHashMap; use itertools::Itertools; -use rattler_conda_types::package::ArchiveIdentifier; -use rattler_conda_types::PackageRecord; -use rattler_networking::AuthenticatedClient; +use rattler_conda_types::{package::ArchiveIdentifier, PackageRecord}; +use rattler_networking::{ + retry_policies::{DoNotRetryPolicy, RetryDecision, RetryPolicy}, + AuthenticatedClient, +}; +use rattler_package_streaming::ExtractError; +use reqwest::StatusCode; use std::error::Error; use std::{ fmt::{Display, Formatter}, @@ -182,12 +187,71 @@ impl PackageCache { pkg: impl Into, url: Url, client: AuthenticatedClient, + ) -> Result { + self.get_or_fetch_from_url_with_retry(pkg, url, client, DoNotRetryPolicy) + .await + } + + /// Returns the directory that contains the specified package. + /// + /// This is a convenience wrapper around `get_or_fetch` which fetches the package from the given + /// URL if the package could not be found in the cache. + pub async fn get_or_fetch_from_url_with_retry( + &self, + pkg: impl Into, + url: Url, + client: AuthenticatedClient, + retry_policy: impl RetryPolicy + Send + 'static, ) -> Result { self.get_or_fetch(pkg, move |destination| async move { - tracing::debug!("downloading {} to {}", &url, destination.display()); - rattler_package_streaming::reqwest::tokio::extract(client, url, &destination) - .await - .map(|_| ()) + let mut current_try = 0; + loop { + current_try += 1; + tracing::debug!("downloading {} to {}", &url, destination.display()); + let result = rattler_package_streaming::reqwest::tokio::extract( + client.clone(), + url.clone(), + &destination, + ) + .await; + + // Extract any potential error + let Err(err) = result else { return Ok(()); }; + + // Only retry on certain errors. + if !matches!( + &err, + ExtractError::IoError(_) | ExtractError::CouldNotCreateDestination(_) + ) && !matches!(&err, ExtractError::ReqwestError(err) if + err.is_timeout() || + err.is_connect() || + err + .status() + .map(|status| status.is_server_error() || status == StatusCode::TOO_MANY_REQUESTS || status == StatusCode::REQUEST_TIMEOUT) + .unwrap_or(false) + ) { + return Err(err); + } + + // Determine whether or not to retry based on the retry policy + let execute_after = match retry_policy.should_retry(current_try) { + RetryDecision::Retry { execute_after } => execute_after, + RetryDecision::DoNotRetry => return Err(err), + }; + let duration = (execute_after - Utc::now()).to_std().expect("the retry duration is out of range"); + + // Wait for a second to let the remote service restore itself. This increases the + // chance of success. + tracing::warn!( + "failed to download and extract {} to {}: {}. Retry #{}, Sleeping {:?} until the next attempt...", + &url, + destination.display(), + err, + current_try, + duration + ); + tokio::time::sleep(duration).await; + } }) .await } @@ -240,9 +304,26 @@ where mod test { use super::PackageCache; use crate::{get_test_data_dir, validation::validate_package_directory}; + use assert_matches::assert_matches; + use axum::{ + extract::State, + http::{Request, StatusCode}, + middleware, + middleware::Next, + response::Response, + routing::get_service, + Router, + }; use rattler_conda_types::package::{ArchiveIdentifier, PackageFile, PathsJson}; - use std::{fs::File, path::Path}; + use rattler_networking::{ + retry_policies::{DoNotRetryPolicy, ExponentialBackoffBuilder}, + AuthenticatedClient, + }; + use std::{fs::File, net::SocketAddr, path::Path, sync::Arc}; use tempfile::tempdir; + use tokio::sync::Mutex; + use tower_http::services::ServeDir; + use url::Url; #[tokio::test] pub async fn test_package_cache() { @@ -284,4 +365,95 @@ mod test { // archive. assert_eq!(current_paths, paths); } + + /// A helper middleware function that fails the first two requests. + async fn fail_the_first_two_requests( + State(count): State>>, + req: Request, + next: Next, + ) -> Result { + let count = { + let mut count = count.lock().await; + *count += 1; + *count + }; + + println!("Running middleware for request #{count} for {}", req.uri()); + if count <= 2 { + println!("Discarding request!"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + + // requires the http crate to get the header name + Ok(next.run(req).await) + } + + #[tokio::test] + pub async fn test_flaky_package_cache() { + let static_dir = get_test_data_dir(); + + // Construct a service that serves raw files from the test directory + let service = get_service(ServeDir::new(static_dir)); + + // Construct a router that returns data from the static dir but fails the first try. + let request_count = Arc::new(Mutex::new(0)); + let router = + Router::new() + .route_service("/*key", service) + .layer(middleware::from_fn_with_state( + request_count.clone(), + fail_the_first_two_requests, + )); + + // Construct the server that will listen on localhost but with a *random port*. The random + // port is very important because it enables creating multiple instances at the same time. + // We need this to be able to run tests in parallel. + let addr = SocketAddr::new([127, 0, 0, 1].into(), 0); + let server = axum::Server::bind(&addr).serve(router.into_make_service()); + + // Get the address of the server so we can bind to it at a later stage. + let addr = server.local_addr(); + + // Spawn the server. + tokio::spawn(server); + + let packages_dir = tempdir().unwrap(); + let cache = PackageCache::new(packages_dir.path()); + + let archive_name = "ros-noetic-rosbridge-suite-0.11.14-py39h6fdeb60_14.tar.bz2"; + let server_url = Url::parse(&format!("http://localhost:{}", addr.port())).unwrap(); + + // Do the first request without + let result = cache + .get_or_fetch_from_url_with_retry( + ArchiveIdentifier::try_from_filename(archive_name).unwrap(), + server_url.join(archive_name).unwrap(), + AuthenticatedClient::default(), + DoNotRetryPolicy, + ) + .await; + + // First request without retry policy should fail + assert_matches!(result, Err(_)); + { + let request_count_lock = request_count.lock().await; + assert_eq!(*request_count_lock, 1, "Expected there to be 1 request"); + } + + // The second one should fail after the 2nd try + let result = cache + .get_or_fetch_from_url_with_retry( + ArchiveIdentifier::try_from_filename(archive_name).unwrap(), + server_url.join(archive_name).unwrap(), + AuthenticatedClient::default(), + ExponentialBackoffBuilder::default().build_with_max_retries(3), + ) + .await; + + assert!(result.is_ok()); + { + let request_count_lock = request_count.lock().await; + assert_eq!(*request_count_lock, 3, "Expected there to be 3 requests"); + } + } } diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml index 0c2e7f111..9d43358a3 100644 --- a/crates/rattler_networking/Cargo.toml +++ b/crates/rattler_networking/Cargo.toml @@ -21,6 +21,7 @@ keyring = "2.0.4" lazy_static = "1.4.0" libc = "0.2.147" reqwest = { version = "0.11.18", features = ["blocking"], default-features = false} +retry-policies = { version = "0.2.0", default-features = false } serde = "1.0.171" serde_json = "1.0.102" thiserror = "1.0.43" diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs index 009f0e535..6da7abfb3 100644 --- a/crates/rattler_networking/src/lib.rs +++ b/crates/rattler_networking/src/lib.rs @@ -8,6 +8,7 @@ pub use authentication_storage::{authentication::Authentication, storage::Authen use reqwest::{Client, IntoUrl, Method, Url}; pub mod authentication_storage; +pub mod retry_policies; /// A client that can be used to make authenticated requests, based on the [`reqwest::Client`] #[derive(Clone)] diff --git a/crates/rattler_networking/src/retry_policies.rs b/crates/rattler_networking/src/retry_policies.rs new file mode 100644 index 000000000..bd3fca2e8 --- /dev/null +++ b/crates/rattler_networking/src/retry_policies.rs @@ -0,0 +1,23 @@ +//! Reexports the trait [`RetryPolicy`] from the `retry_policies` crate as well as all +//! implementations. +//! +//! This module also provides the [`DoNotRetryPolicy`] which is useful if you do not want to retry +//! anything. + +pub use retry_policies::{policies::*, Jitter, RetryDecision, RetryPolicy}; + +/// A simple [`RetryPolicy`] that just never retries. +pub struct DoNotRetryPolicy; +impl RetryPolicy for DoNotRetryPolicy { + fn should_retry(&self, _: u32) -> RetryDecision { + RetryDecision::DoNotRetry + } +} + +/// Returns the default retry policy that can be used . +/// +/// This is useful if you just do not care about a retry policy and you just want something +/// sensible. Note that the behavior of what is "sensible" might change over time. +pub fn default_retry_policy() -> ExponentialBackoff { + ExponentialBackoff::builder().build_with_max_retries(3) +} From 856b70fba5c2aa833253abcd8d7d9ac2ded5f44f Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 22 Aug 2023 16:09:13 +0200 Subject: [PATCH 3/8] bump: CHANGELOG.md --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 799d96326..1d8c927d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.8.0] - 2023-08-22 + +### Highlights + +This release contains bug fixes. + +### Details + +#### Added + +- retry behavior when downloading package archives by @baszalmstra in ([#281](https://github.com/mamba-org/rattler/pull/281)) + +#### Fixed + +- parsing of local versions in `Constraint`s by @baszalmstra in ([#280](https://github.com/mamba-org/rattler/pull/280)) ## [0.7.0] - 2023-08-11 From 1f8edca79ef7b59016864f89e1ff4a592154e9a1 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 22 Aug 2023 16:10:13 +0200 Subject: [PATCH 4/8] chore: Release --- Cargo.toml | 2 +- crates/rattler-bin/Cargo.toml | 12 ++++++------ crates/rattler/Cargo.toml | 8 ++++---- crates/rattler_conda_types/Cargo.toml | 4 ++-- crates/rattler_libsolv_rs/Cargo.toml | 2 +- crates/rattler_package_streaming/Cargo.toml | 6 +++--- crates/rattler_repodata_gateway/Cargo.toml | 6 +++--- crates/rattler_shell/Cargo.toml | 2 +- crates/rattler_solve/Cargo.toml | 10 +++++----- crates/rattler_virtual_packages/Cargo.toml | 2 +- 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f8a3135d8..d11b95e30 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ opt-level = 3 opt-level = 3 [workspace.package] -version = "0.7.0" +version = "0.8.0" categories = ["conda"] homepage = "https://github.com/mamba-org/rattler" repository = "https://github.com/mamba-org/rattler" diff --git a/crates/rattler-bin/Cargo.toml b/crates/rattler-bin/Cargo.toml index 748b4fe08..0a1e72428 100644 --- a/crates/rattler-bin/Cargo.toml +++ b/crates/rattler-bin/Cargo.toml @@ -29,12 +29,12 @@ futures = "0.3.28" indicatif = "0.17.5" itertools = "0.11.0" once_cell = "1.18.0" -rattler = { version = "0.7.0", path = "../rattler", default-features = false } -rattler_networking = { version = "0.7.0", path = "../rattler_networking", default-features = false } -rattler_conda_types = { version = "0.7.0", path = "../rattler_conda_types" } -rattler_repodata_gateway = { version = "0.7.0", path = "../rattler_repodata_gateway", features = ["sparse"], default-features = false } -rattler_solve = { version = "0.7.0", path = "../rattler_solve", features = ["libsolv_rs", "libsolv_c"] } -rattler_virtual_packages = { version = "0.7.0", path = "../rattler_virtual_packages" } +rattler = { version = "0.8.0", path = "../rattler", default-features = false } +rattler_networking = { version = "0.8.0", path = "../rattler_networking", default-features = false } +rattler_conda_types = { version = "0.8.0", path = "../rattler_conda_types" } +rattler_repodata_gateway = { version = "0.8.0", path = "../rattler_repodata_gateway", features = ["sparse"], default-features = false } +rattler_solve = { version = "0.8.0", path = "../rattler_solve", features = ["libsolv_rs", "libsolv_c"] } +rattler_virtual_packages = { version = "0.8.0", path = "../rattler_virtual_packages" } reqwest = { version = "0.11.18", default-features = false } tokio = { version = "1.29.1", features = ["rt-multi-thread", "macros"] } tracing-subscriber = { version = "0.3.17", features = ["env-filter"] } diff --git a/crates/rattler/Cargo.toml b/crates/rattler/Cargo.toml index 50c9c346c..4f81df8bc 100644 --- a/crates/rattler/Cargo.toml +++ b/crates/rattler/Cargo.toml @@ -31,10 +31,10 @@ memmap2 = "0.7.1" nom = "7.1.3" once_cell = "1.18.0" pin-project-lite = "0.2.10" -rattler_conda_types = { version = "0.7.0", path = "../rattler_conda_types" } -rattler_digest = { version = "0.7.0", path = "../rattler_digest" } -rattler_networking = { version = "0.7.0", path = "../rattler_networking", default-features = false } -rattler_package_streaming = { version = "0.7.0", path = "../rattler_package_streaming", features = ["reqwest", "tokio"], default-features = false } +rattler_conda_types = { version = "0.8.0", path = "../rattler_conda_types" } +rattler_digest = { version = "0.8.0", path = "../rattler_digest" } +rattler_networking = { version = "0.8.0", path = "../rattler_networking", default-features = false } +rattler_package_streaming = { version = "0.8.0", path = "../rattler_package_streaming", features = ["reqwest", "tokio"], default-features = false } regex = "1.9.1" reqwest = { version = "0.11.18", default-features = false, features = ["stream", "json", "gzip"] } serde = { version = "1.0.171", features = ["derive"] } diff --git a/crates/rattler_conda_types/Cargo.toml b/crates/rattler_conda_types/Cargo.toml index 029445f1f..58679aaf5 100644 --- a/crates/rattler_conda_types/Cargo.toml +++ b/crates/rattler_conda_types/Cargo.toml @@ -29,8 +29,8 @@ strum = { version = "0.25.0", features = ["derive"] } thiserror = "1.0.43" tracing = "0.1.37" url = { version = "2.4.0", features = ["serde"] } -rattler_digest = { version = "0.7.0", path = "../rattler_digest", features = ["serde"] } -rattler_macros = { version = "0.7.0", path = "../rattler_macros" } +rattler_digest = { version = "0.8.0", path = "../rattler_digest", features = ["serde"] } +rattler_macros = { version = "0.8.0", path = "../rattler_macros" } glob = "0.3.1" [dev-dependencies] diff --git a/crates/rattler_libsolv_rs/Cargo.toml b/crates/rattler_libsolv_rs/Cargo.toml index 252314866..1f21e243d 100644 --- a/crates/rattler_libsolv_rs/Cargo.toml +++ b/crates/rattler_libsolv_rs/Cargo.toml @@ -13,7 +13,7 @@ readme.workspace = true [dependencies] itertools = "0.11.0" petgraph = "0.6.3" -rattler_conda_types = { version = "0.7.0", path = "../rattler_conda_types" } +rattler_conda_types = { version = "0.8.0", path = "../rattler_conda_types" } tracing = "0.1.37" [dev-dependencies] diff --git a/crates/rattler_package_streaming/Cargo.toml b/crates/rattler_package_streaming/Cargo.toml index a319908e6..9f86bc7f4 100644 --- a/crates/rattler_package_streaming/Cargo.toml +++ b/crates/rattler_package_streaming/Cargo.toml @@ -15,8 +15,8 @@ bzip2 = { version = "0.4" } chrono = "0.4.26" futures-util = { version = "0.3.28", optional = true } itertools = "0.11.0" -rattler_conda_types = { version = "0.7.0", path = "../rattler_conda_types" } -rattler_digest = { version = "0.7.0", path = "../rattler_digest" } +rattler_conda_types = { version = "0.8.0", path = "../rattler_conda_types" } +rattler_digest = { version = "0.8.0", path = "../rattler_digest" } serde_json = "1.0.102" tar = { version = "0.4.38" } thiserror = "1.0.43" @@ -26,7 +26,7 @@ reqwest = { version = "0.11.18", optional = true, default-features = false } url = "2.4.0" zip = { version = "0.6.6" } zstd = "0.12.3" -rattler_networking = { version = "0.7.0", path = "../rattler_networking", default-features = false } +rattler_networking = { version = "0.8.0", path = "../rattler_networking", default-features = false } [features] default = ['native-tls'] diff --git a/crates/rattler_repodata_gateway/Cargo.toml b/crates/rattler_repodata_gateway/Cargo.toml index 1448384c3..0bb1291a9 100644 --- a/crates/rattler_repodata_gateway/Cargo.toml +++ b/crates/rattler_repodata_gateway/Cargo.toml @@ -30,8 +30,8 @@ serde = { version = "1.0.171", features = ["derive"] } serde_json = { version = "1.0.102" } pin-project-lite = "0.2.10" md-5 = "0.10.5" -rattler_digest = { version = "0.7.0", path = "../rattler_digest", features = ["tokio", "serde"] } -rattler_conda_types = { version = "0.7.0", path = "../rattler_conda_types", optional = true } +rattler_digest = { version = "0.8.0", path = "../rattler_digest", features = ["tokio", "serde"] } +rattler_conda_types = { version = "0.8.0", path = "../rattler_conda_types", optional = true } fxhash = { version = "0.2.1", optional = true } memmap2 = { version = "0.7.1", optional = true } ouroboros = { version = "0.17.0", optional = true } @@ -40,7 +40,7 @@ superslice = { version = "1.0.0", optional = true } itertools = { version = "0.11.0", optional = true } json-patch = "1.0.0" hex = { version = "0.4.3", features = ["serde"] } -rattler_networking = { version = "0.7.0", path = "../rattler_networking", default-features = false } +rattler_networking = { version = "0.8.0", path = "../rattler_networking", default-features = false } [target.'cfg(unix)'.dependencies] libc = "0.2" diff --git a/crates/rattler_shell/Cargo.toml b/crates/rattler_shell/Cargo.toml index 8b3502f22..a4fae659c 100644 --- a/crates/rattler_shell/Cargo.toml +++ b/crates/rattler_shell/Cargo.toml @@ -14,7 +14,7 @@ readme.workspace = true enum_dispatch = "0.3.12" indexmap = "2.0.0" itertools = "0.11.0" -rattler_conda_types = { version = "0.7.0", path = "../rattler_conda_types" } +rattler_conda_types = { version = "0.8.0", path = "../rattler_conda_types" } serde_json = { version = "1.0.102", features = ["preserve_order"] } shlex = "1.1.0" sysinfo = { version = "0.29.4", optional = true } diff --git a/crates/rattler_solve/Cargo.toml b/crates/rattler_solve/Cargo.toml index aa2a05c80..30c43ee8e 100644 --- a/crates/rattler_solve/Cargo.toml +++ b/crates/rattler_solve/Cargo.toml @@ -11,8 +11,8 @@ license.workspace = true readme.workspace = true [dependencies] -rattler_conda_types = { version = "0.7.0", path = "../rattler_conda_types" } -rattler_digest = { version = "0.7.0", path = "../rattler_digest" } +rattler_conda_types = { version = "0.8.0", path = "../rattler_conda_types" } +rattler_digest = { version = "0.8.0", path = "../rattler_digest" } libc = { version = "0.2", optional = true } anyhow = "1.0.71" thiserror = "1.0.43" @@ -21,11 +21,11 @@ serde = { version = "1.0.171", features = ["derive"] } url = "2.4.0" hex = "0.4.3" tempfile = "3.6.0" -rattler_libsolv_c = { version = "0.7.0", path = "../rattler_libsolv_c", optional = true } -rattler_libsolv_rs = { version = "0.7.0", path = "../rattler_libsolv_rs", optional = true } +rattler_libsolv_c = { version = "0.8.0", path = "../rattler_libsolv_c", optional = true } +rattler_libsolv_rs = { version = "0.8.0", path = "../rattler_libsolv_rs", optional = true } [dev-dependencies] -rattler_repodata_gateway = { version = "0.7.0", path = "../rattler_repodata_gateway", default-features = false, features = ["sparse"] } +rattler_repodata_gateway = { version = "0.8.0", path = "../rattler_repodata_gateway", default-features = false, features = ["sparse"] } insta = { version = "1.30.0", features = ["yaml"] } rstest = "0.18.1" serde_json = "1.0.102" diff --git a/crates/rattler_virtual_packages/Cargo.toml b/crates/rattler_virtual_packages/Cargo.toml index b4ff0fca8..28d0e76df 100644 --- a/crates/rattler_virtual_packages/Cargo.toml +++ b/crates/rattler_virtual_packages/Cargo.toml @@ -15,7 +15,7 @@ cfg-if = "1.0.0" libloading = "0.8.0" nom = "7.1.3" once_cell = "1.18.0" -rattler_conda_types = { version = "0.7.0", path = "../rattler_conda_types" } +rattler_conda_types = { version = "0.8.0", path = "../rattler_conda_types" } thiserror = "1.0.43" tracing = "0.1.37" serde = { version = "1.0.171", features = ["derive"] } From 313684edacabde3db6be68f6d773485900a4950c Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 22 Aug 2023 22:42:15 +0200 Subject: [PATCH 5/8] fix: typo in solver error message (#284) fix: typo in error message --- crates/rattler_libsolv_rs/src/problem.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/rattler_libsolv_rs/src/problem.rs b/crates/rattler_libsolv_rs/src/problem.rs index d8c1f89cb..b5b7d56ae 100644 --- a/crates/rattler_libsolv_rs/src/problem.rs +++ b/crates/rattler_libsolv_rs/src/problem.rs @@ -561,9 +561,9 @@ impl<'a> DisplayUnsat<'a> { if missing { // No candidates for requirement if depth == 0 { - writeln!(f, "{indent}No candidates where found for {req}.")?; + writeln!(f, "{indent}No candidates were found for {req}.")?; } else { - writeln!(f, "{indent}{req}, for which no candidates where found.",)?; + writeln!(f, "{indent}{req}, for which no candidates were found.",)?; } } else if installable { // Package can be installed (only mentioned for top-level requirements) From 40cccb866cadde0dfcccf425e27a7bdf54ebde43 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra Date: Tue, 22 Aug 2023 23:27:02 +0200 Subject: [PATCH 6/8] hotfix: snapshots --- ...libsolv_rs__solver__test__unsat_missing_top_level_dep_1.snap | 2 +- ...libsolv_rs__solver__test__unsat_missing_top_level_dep_2.snap | 2 +- ...bsolv_rs__solver__test__unsat_no_candidates_for_child_1.snap | 2 +- ...bsolv_rs__solver__test__unsat_no_candidates_for_child_2.snap | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_missing_top_level_dep_1.snap b/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_missing_top_level_dep_1.snap index 415bcb27f..8ef62b604 100644 --- a/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_missing_top_level_dep_1.snap +++ b/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_missing_top_level_dep_1.snap @@ -2,5 +2,5 @@ source: crates/rattler_libsolv_rs/src/solver/mod.rs expression: error --- -No candidates where found for fghj. +No candidates were found for fghj. diff --git a/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_missing_top_level_dep_2.snap b/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_missing_top_level_dep_2.snap index a954a6b8a..0ee601109 100644 --- a/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_missing_top_level_dep_2.snap +++ b/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_missing_top_level_dep_2.snap @@ -2,5 +2,5 @@ source: crates/rattler_libsolv_rs/src/solver/mod.rs expression: error --- -No candidates where found for B 14.*. +No candidates were found for B 14.*. diff --git a/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_no_candidates_for_child_1.snap b/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_no_candidates_for_child_1.snap index 75165dea7..e2616024b 100644 --- a/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_no_candidates_for_child_1.snap +++ b/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_no_candidates_for_child_1.snap @@ -4,5 +4,5 @@ expression: error --- asdf cannot be installed because there are no viable options: |-- asdf 1.2.3 would require - |-- C >1, for which no candidates where found. + |-- C >1, for which no candidates were found. diff --git a/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_no_candidates_for_child_2.snap b/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_no_candidates_for_child_2.snap index f732360b4..8cd476b60 100644 --- a/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_no_candidates_for_child_2.snap +++ b/crates/rattler_libsolv_rs/src/solver/snapshots/rattler_libsolv_rs__solver__test__unsat_no_candidates_for_child_2.snap @@ -4,5 +4,5 @@ expression: error --- A <1000 cannot be installed because there are no viable options: |-- A 41 would require - |-- B <20, for which no candidates where found. + |-- B <20, for which no candidates were found. From 628b83cf8cc3cbfdf12b7f57325daa190c5ba095 Mon Sep 17 00:00:00 2001 From: Tarun Pratap Singh <101409098+Wackyator@users.noreply.github.com> Date: Wed, 23 Aug 2023 12:44:56 +0530 Subject: [PATCH 7/8] feat: expose ParseMatchSpecError in rattler_conda_types (#286) --- crates/rattler_conda_types/src/lib.rs | 1 + crates/rattler_conda_types/src/match_spec/parse.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+) diff --git a/crates/rattler_conda_types/src/lib.rs b/crates/rattler_conda_types/src/lib.rs index e2e5dd89a..5c2d8a928 100644 --- a/crates/rattler_conda_types/src/lib.rs +++ b/crates/rattler_conda_types/src/lib.rs @@ -28,6 +28,7 @@ pub use explicit_environment_spec::{ }; pub use generic_virtual_package::GenericVirtualPackage; pub use match_spec::matcher::StringMatcher; +pub use match_spec::parse::ParseMatchSpecError; pub use match_spec::{MatchSpec, NamelessMatchSpec}; pub use no_arch_type::{NoArchKind, NoArchType}; pub use platform::{ParsePlatformError, Platform}; diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs index 5b72fbcc3..c4759e3f2 100644 --- a/crates/rattler_conda_types/src/match_spec/parse.rs +++ b/crates/rattler_conda_types/src/match_spec/parse.rs @@ -21,41 +21,54 @@ use std::str::FromStr; use thiserror::Error; use url::Url; +/// The type of parse error that occurred when parsing match spec. #[derive(Debug, Clone, Eq, PartialEq, Error)] pub enum ParseMatchSpecError { + /// The path or url of the package was invalid #[error("invalid package path or url")] InvalidPackagePathOrUrl, + /// Invalid bracket in match spec #[error("invalid bracket")] InvalidBracket, + /// Invalid number of colons in match spec #[error("invalid number of colons")] InvalidNumberOfColons, + /// Invalid channel provided in match spec #[error("invalid channel")] ParseChannelError(#[from] ParseChannelError), + /// Invalid key in match spec #[error("invalid bracket key: {0}")] InvalidBracketKey(String), + /// Missing package name in match spec #[error("missing package name")] MissingPackageName, + /// Multiple bracket sections in match spec #[error("multiple bracket sections not allowed")] MultipleBracketSectionsNotAllowed, + /// Invalid version and build #[error("Unable to parse version spec: {0}")] InvalidVersionAndBuild(String), + /// Invalid version spec #[error("invalid version spec: {0}")] InvalidVersionSpec(#[from] ParseVersionSpecError), + /// Invalid string matcher #[error("invalid string matcher: {0}")] InvalidStringMatcher(#[from] StringMatcherParseError), + /// Invalid build number #[error("invalid build number: {0}")] InvalidBuildNumber(#[from] ParseIntError), + /// Unable to parse hash digest from hex #[error("Unable to parse hash digest from hex")] InvalidHashDigest, } From fc88e125c7b274df9374dcf4e5ab6e5689fd3e33 Mon Sep 17 00:00:00 2001 From: Orion Yeung <11580988+YeungOnion@users.noreply.github.com> Date: Wed, 23 Aug 2023 07:00:12 -0500 Subject: [PATCH 8/8] address issue #282 (#283) * add as pixi project, mention in contributing readme added pixi.toml with depends as well as build tasks, made CONTRIBUTING.md mostly added stuff regarding dev. * Update pixi.toml for other platforms Co-authored-by: Bas Zalmstra * Update pixi.toml for fmt and clippy Co-authored-by: Bas Zalmstra * Update CONTRIBUTING.md Co-authored-by: Bas Zalmstra * Update pixi.toml for approx version syntax Co-authored-by: Bas Zalmstra * Update CONTRIBUTING.md Co-authored-by: Bas Zalmstra * Update README.md Co-authored-by: Bas Zalmstra * Update pixi.toml Co-authored-by: Bas Zalmstra * Update CONTRIBUTING.md Co-authored-by: Bas Zalmstra --------- Co-authored-by: Orion Yeung <11580988+orionyeung001@users.noreply.github.com> Co-authored-by: Bas Zalmstra --- .gitignore | 4 ++++ CONTRIBUTING.md | 31 +++++++++++++++++++++++++++++++ README.md | 6 ++---- pixi.toml | 20 ++++++++++++++++++++ 4 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 CONTRIBUTING.md create mode 100644 pixi.toml diff --git a/.gitignore b/.gitignore index 7df89a2fc..38f662d90 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ Cargo.lock # rattler .prefix + +# pixi +.pixi/ +pixi.lock diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..85ba0b8ef --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing 😍 + +We would love to have you contribute! +For a good list of things you could help us with, take a look at our [*good first issues*](https://github.com/mamba-org/rattler/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). +If you want to go deeper though, any [open issue](https://github.com/mamba-org/rattler/issues) is up for grabs. +Just let us know what you start on something. + +For questions, requests or a casual chat, we are very active on our discord server. +You can [join our discord server via this link][chat-url]. + +## Development +If you'd like to contribute code, then you may want to manage the build depends with a tool, we suggest pixi, but conda/mamba will also work. + +### Virtual env with pixi +You can use [pixi](https://github.com/prefix-dev/pixi) for setting up the environment needed for building and testing rattler, (as a fun fact, pixi uses rattler as a dependency!). The spec in `pixi.toml` in the project root will set up the environment. After installing, run the install command from the project root directory, shown below. +```sh +❱ pixi install # installs dependencies into the virtual env +❱ pixi run build # calls "build" task specified in pixi.toml, "cargo build", using cargo in pixi venv +``` + +### Virtual env with conda/mamba +The environment can also be managed with conda using the spec in `environments.yml` in the project root. +As below, +```sh +❱ mamba create -n name_of_your_rattler_env --file='environments.yml' && mamba activate name_of_your_rattler_env +❱ cargo build # uses cargo from your mamba venv +❱ mamba deactivate # don't forget you're in the venv +``` + + + diff --git a/README.md b/README.md index 02b93f79c..eaeecea87 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,9 @@ Try it! ## Contributing 😍 We would love to have you contribute! -For a good list of things you could help us with, take a look at our [*good first issues*](https://github.com/mamba-org/rattler/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). -If you want to go deeper though, any [open issue](https://github.com/mamba-org/rattler/issues) is up for grabs. -Just let us know what you start on something. +See the CONTRIBUTION.md for more info. For questions, requests or a casual chat, we are very active on our discord server. +You can [join our discord server via this link][chat-url]. -For questions, requests or a casual chat, we are very active on our discord server. You can [join our discord server via this link][chat-url]. ## Components diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 000000000..6b7c4034f --- /dev/null +++ b/pixi.toml @@ -0,0 +1,20 @@ +[project] +name = "rattler" +version = "0.7.0" +description = "Rust library to install conda environments" +authors = ["Wolf Vollprecht ", "Bas Zalmstra ", "Tim de Jager ", "Ruben Arts "] +channels = ["conda-forge"] +platforms = ["linux-64", "win-64", "osx-64", "osx-arm64"] + +[tasks] +build = "cargo build" +fmt = "cargo fmt" +lint = "cargo clippy" + +[dependencies] +cxx-compiler = "~=1.6.0" +openssl = "~=3.1.2" +make = "~=4.3" +pkg-config = "~=0.29.2" +rust = "~=1.71.1" +cmake = "~=3.26.4"