diff --git a/Cargo.toml b/Cargo.toml index 657650385..e49799286 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["crates/*"] +members = ["crates/*", "tools/*"] resolver = "2" # See: https://docs.rs/insta/latest/insta/#optional-faster-runs @@ -108,7 +108,7 @@ regex = "1.10.4" reqwest = { version = "0.12.3", default-features = false } reqwest-middleware = "0.3.0" reqwest-retry = "0.5.0" -resolvo = { version = "0.6.0" } +resolvo = { version = "0.6.1" } retry-policies = { version = "0.3.0", default-features = false } rmp-serde = { version = "1.2.0" } rstest = { version = "0.19.0" } @@ -153,6 +153,3 @@ walkdir = "2.5.0" windows-sys = { version = "0.52.0", default-features = false } zip = { version = "0.6.6", default-features = false } zstd = { version = "0.13.1", default-features = false } - -[profile.release] -debug = true diff --git a/crates/rattler_solve/src/resolvo/mod.rs b/crates/rattler_solve/src/resolvo/mod.rs index 4261869f4..ed53cd7d8 100644 --- a/crates/rattler_solve/src/resolvo/mod.rs +++ b/crates/rattler_solve/src/resolvo/mod.rs @@ -155,9 +155,11 @@ impl<'a> Display for SolverPackageRecord<'a> { } } -/// Dependency provider for conda +/// An implement of [`resolvo::DependencyProvider`] that implements the +/// ecosystem behavior for conda. This allows resolvo to solve for conda +/// packages. #[derive(Default)] -pub(crate) struct CondaDependencyProvider<'a> { +pub struct CondaDependencyProvider<'a> { pool: Pool, String>, records: HashMap, @@ -175,8 +177,9 @@ pub(crate) struct CondaDependencyProvider<'a> { } impl<'a> CondaDependencyProvider<'a> { + /// Constructs a new provider. #[allow(clippy::too_many_arguments)] - pub fn from_solver_task( + pub fn new( repodata: impl IntoIterator>, favored_records: &'a [RepoDataRecord], locked_records: &'a [RepoDataRecord], @@ -397,6 +400,11 @@ impl<'a> CondaDependencyProvider<'a> { direct_dependencies, }) } + + /// Returns all package names + pub fn package_names(&self) -> impl Iterator + '_ { + self.records.keys().copied() + } } /// The reason why the solver was cancelled @@ -590,7 +598,7 @@ impl super::SolverImpl for Solver { .map(|timeout| std::time::SystemTime::now() + timeout); // Construct a provider that can serve the data. - let provider = CondaDependencyProvider::from_solver_task( + let provider = CondaDependencyProvider::new( task.available_packages.into_iter().map(|r| r.into()), &task.locked_packages, &task.pinned_packages, diff --git a/py-rattler/Cargo.lock b/py-rattler/Cargo.lock index d89d2d55a..09bf851c5 100644 --- a/py-rattler/Cargo.lock +++ b/py-rattler/Cargo.lock @@ -2997,9 +2997,9 @@ dependencies = [ [[package]] name = "resolvo" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e3520f5062db83765158b91a17d01c8309feab0a8327e0991cd7bb33e9d5fdb" +checksum = "ee094f3d073e5efe649c7f24c237657fa4c613eb9becca896ed682f31ca389c2" dependencies = [ "ahash", "bitvec", diff --git a/tools/create-resolvo-snapshot/Cargo.toml b/tools/create-resolvo-snapshot/Cargo.toml new file mode 100644 index 000000000..5cd8db1e7 --- /dev/null +++ b/tools/create-resolvo-snapshot/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "create-resolvo-snapshot" +version = "0.1.0" +description = "Create a resolvo snapshot of a conda channel" +homepage.workspace = true +repository.workspace = true +license.workspace = true +edition.workspace = true +readme = "README.md" +publish = false + +[dependencies] +clap = { workspace = true } +itertools = { workspace = true } +rattler_cache = { path = "../../crates/rattler_cache" } +rattler_conda_types = { path = "../../crates/rattler_conda_types" } +rattler_repodata_gateway = { path = "../../crates/rattler_repodata_gateway", default-features = false, features = ["rustls-tls"] } +rattler_solve = { path = "../../crates/rattler_solve" } +reqwest = { workspace = true, default-features = false, features = ["rustls-tls"] } +resolvo = { workspace = true, features = ["serde"] } +serde_json = { workspace = true } +tokio = { workspace = true, features = ["rt-multi-thread"] } diff --git a/tools/create-resolvo-snapshot/README.md b/tools/create-resolvo-snapshot/README.md new file mode 100644 index 000000000..0525b1765 --- /dev/null +++ b/tools/create-resolvo-snapshot/README.md @@ -0,0 +1,9 @@ +# Create-resolvo-snapshot + +A tool to create a resolvo snapshot from a conda channel. + +#### How-to-use + +``` +create-resolvo-snapshot conda-forge --subdir linux-64 +``` diff --git a/tools/create-resolvo-snapshot/src/main.rs b/tools/create-resolvo-snapshot/src/main.rs new file mode 100644 index 000000000..96b3d4d43 --- /dev/null +++ b/tools/create-resolvo-snapshot/src/main.rs @@ -0,0 +1,106 @@ +use std::{collections::HashSet, io::BufWriter, path::Path}; + +use clap::Parser; +use itertools::Itertools; +use rattler_conda_types::{Channel, ChannelConfig, Platform}; +use rattler_repodata_gateway::fetch::FetchRepoDataOptions; +use rattler_solve::{ChannelPriority, SolveStrategy}; +use reqwest::Client; + +#[derive(Parser)] +#[clap(about)] +struct Args { + /// The channel to make the snapshot for. + channel: String, + + /// The subdirs to query. + #[clap(short, long, num_args=1..)] + subdir: Vec, + + /// The output path + #[clap(short)] + output: Option, +} + +#[tokio::main] +async fn main() { + let args: Args = Args::parse(); + + // Determine the channel + let channel = Channel::from_str( + &args.channel, + &ChannelConfig::default_with_root_dir(std::env::current_dir().unwrap()), + ) + .unwrap(); + + // Fetch the repodata for all the subdirs. + let mut subdirs: HashSet = HashSet::from_iter(args.subdir); + if subdirs.is_empty() { + subdirs.insert(Platform::current()); + } + subdirs.insert(Platform::NoArch); + + let client = Client::default(); + let mut records = Vec::new(); + for &subdir in &subdirs { + eprintln!("fetching repodata for {subdir:?}.."); + let repodata = rattler_repodata_gateway::fetch::fetch_repo_data( + channel.platform_url(subdir), + client.clone().into(), + rattler_cache::default_cache_dir().unwrap().join("repodata"), + FetchRepoDataOptions::default(), + None, + ) + .await + .unwrap(); + + eprintln!("parsing repodata.."); + let repodata = rattler_conda_types::RepoData::from_path(repodata.repo_data_json_path) + .unwrap() + .into_repo_data_records(&channel); + + records.push(repodata); + } + + // Create the dependency provider + let provider = rattler_solve::resolvo::CondaDependencyProvider::new( + records + .iter() + .map(rattler_solve::resolvo::RepoData::from_iter), + &[], + &[], + &[], + &[], + None, + ChannelPriority::default(), + None, + SolveStrategy::default(), + ) + .unwrap(); + + eprintln!("creating snapshot.."); + let package_names = provider.package_names().collect::>(); + let snapshot = + resolvo::snapshot::DependencySnapshot::from_provider(provider, package_names, [], []) + .unwrap(); + + let output_file = args.output.unwrap_or_else(|| { + format!( + "snapshot-{}-{}.json", + channel.name(), + subdirs + .iter() + .copied() + .map(Platform::as_str) + .sorted() + .join("-") + ) + }); + eprintln!("serializing snapshot to {}", &output_file); + let snapshot_path = Path::new(&output_file); + if let Some(dir) = snapshot_path.parent() { + std::fs::create_dir_all(dir).unwrap(); + } + let snapshot_file = BufWriter::new(std::fs::File::create(snapshot_path).unwrap()); + serde_json::to_writer(snapshot_file, &snapshot).unwrap(); +}