feat: add tool to generate resolvo snapshots (#741)

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = ["crates/*"]
+members = ["crates/*", "tools/*"]
 resolver = "2"
 
 # See: https://docs.rs/insta/latest/insta/#optional-faster-runs
@@ -108,7 +108,7 @@ regex = "1.10.4"
 reqwest = { version = "0.12.3", default-features = false }
 reqwest-middleware = "0.3.0"
 reqwest-retry = "0.5.0"
-resolvo = { version = "0.6.0" }
+resolvo = { version = "0.6.1" }
 retry-policies = { version = "0.3.0", default-features = false }
 rmp-serde = { version = "1.2.0" }
 rstest = { version = "0.19.0" }
@@ -153,6 +153,3 @@ walkdir = "2.5.0"
 windows-sys = { version = "0.52.0", default-features = false }
 zip = { version = "0.6.6", default-features = false }
 zstd = { version = "0.13.1", default-features = false }
-
-[profile.release]
-debug = true

crates/rattler_solve/src/resolvo/mod.rs

@@ -155,9 +155,11 @@ impl<'a> Display for SolverPackageRecord<'a> {
     }
 }
 
-/// Dependency provider for conda
+/// An implement of [`resolvo::DependencyProvider`] that implements the
+/// ecosystem behavior for conda. This allows resolvo to solve for conda
+/// packages.
 #[derive(Default)]
-pub(crate) struct CondaDependencyProvider<'a> {
+pub struct CondaDependencyProvider<'a> {
     pool: Pool<SolverMatchSpec<'a>, String>,
 
     records: HashMap<NameId, Candidates>,
@@ -175,8 +177,9 @@ pub(crate) struct CondaDependencyProvider<'a> {
 }
 
 impl<'a> CondaDependencyProvider<'a> {
+    /// Constructs a new provider.
     #[allow(clippy::too_many_arguments)]
-    pub fn from_solver_task(
+    pub fn new(
         repodata: impl IntoIterator<Item = RepoData<'a>>,
         favored_records: &'a [RepoDataRecord],
         locked_records: &'a [RepoDataRecord],
@@ -397,6 +400,11 @@ impl<'a> CondaDependencyProvider<'a> {
             direct_dependencies,
         })
     }
+
+    /// Returns all package names
+    pub fn package_names(&self) -> impl Iterator<Item = NameId> + '_ {
+        self.records.keys().copied()
+    }
 }
 
 /// The reason why the solver was cancelled
@@ -590,7 +598,7 @@ impl super::SolverImpl for Solver {
             .map(|timeout| std::time::SystemTime::now() + timeout);
 
         // Construct a provider that can serve the data.
-        let provider = CondaDependencyProvider::from_solver_task(
+        let provider = CondaDependencyProvider::new(
             task.available_packages.into_iter().map(|r| r.into()),
             &task.locked_packages,
             &task.pinned_packages,

tools/create-resolvo-snapshot/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "create-resolvo-snapshot"
+version = "0.1.0"
+description = "Create a resolvo snapshot of a conda channel"
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+edition.workspace = true
+readme = "README.md"
+publish = false
+
+[dependencies]
+clap = { workspace = true }
+itertools = { workspace = true }
+rattler_cache = { path = "../../crates/rattler_cache" }
+rattler_conda_types = { path = "../../crates/rattler_conda_types" }
+rattler_repodata_gateway = { path = "../../crates/rattler_repodata_gateway", default-features = false, features = ["rustls-tls"] }
+rattler_solve = { path = "../../crates/rattler_solve" }
+reqwest = { workspace = true, default-features = false, features = ["rustls-tls"] }
+resolvo = { workspace = true, features = ["serde"] }
+serde_json = { workspace = true }
+tokio = { workspace = true, features = ["rt-multi-thread"] }

tools/create-resolvo-snapshot/README.md

@@ -0,0 +1,9 @@
+# Create-resolvo-snapshot
+
+A tool to create a resolvo snapshot from a conda channel. 
+
+#### How-to-use
+
+```
+create-resolvo-snapshot conda-forge --subdir linux-64
+```

tools/create-resolvo-snapshot/src/main.rs

@@ -0,0 +1,106 @@
+use std::{collections::HashSet, io::BufWriter, path::Path};
+
+use clap::Parser;
+use itertools::Itertools;
+use rattler_conda_types::{Channel, ChannelConfig, Platform};
+use rattler_repodata_gateway::fetch::FetchRepoDataOptions;
+use rattler_solve::{ChannelPriority, SolveStrategy};
+use reqwest::Client;
+
+#[derive(Parser)]
+#[clap(about)]
+struct Args {
+    /// The channel to make the snapshot for.
+    channel: String,
+
+    /// The subdirs to query.
+    #[clap(short, long, num_args=1..)]
+    subdir: Vec<Platform>,
+
+    /// The output path
+    #[clap(short)]
+    output: Option<String>,
+}
+
+#[tokio::main]
+async fn main() {
+    let args: Args = Args::parse();
+
+    // Determine the channel
+    let channel = Channel::from_str(
+        &args.channel,
+        &ChannelConfig::default_with_root_dir(std::env::current_dir().unwrap()),
+    )
+    .unwrap();
+
+    // Fetch the repodata for all the subdirs.
+    let mut subdirs: HashSet<Platform> = HashSet::from_iter(args.subdir);
+    if subdirs.is_empty() {
+        subdirs.insert(Platform::current());
+    }
+    subdirs.insert(Platform::NoArch);
+
+    let client = Client::default();
+    let mut records = Vec::new();
+    for &subdir in &subdirs {
+        eprintln!("fetching repodata for {subdir:?}..");
+        let repodata = rattler_repodata_gateway::fetch::fetch_repo_data(
+            channel.platform_url(subdir),
+            client.clone().into(),
+            rattler_cache::default_cache_dir().unwrap().join("repodata"),
+            FetchRepoDataOptions::default(),
+            None,
+        )
+        .await
+        .unwrap();
+
+        eprintln!("parsing repodata..");
+        let repodata = rattler_conda_types::RepoData::from_path(repodata.repo_data_json_path)
+            .unwrap()
+            .into_repo_data_records(&channel);
+
+        records.push(repodata);
+    }
+
+    // Create the dependency provider
+    let provider = rattler_solve::resolvo::CondaDependencyProvider::new(
+        records
+            .iter()
+            .map(rattler_solve::resolvo::RepoData::from_iter),
+        &[],
+        &[],
+        &[],
+        &[],
+        None,
+        ChannelPriority::default(),
+        None,
+        SolveStrategy::default(),
+    )
+    .unwrap();
+
+    eprintln!("creating snapshot..");
+    let package_names = provider.package_names().collect::<Vec<_>>();
+    let snapshot =
+        resolvo::snapshot::DependencySnapshot::from_provider(provider, package_names, [], [])
+            .unwrap();
+
+    let output_file = args.output.unwrap_or_else(|| {
+        format!(
+            "snapshot-{}-{}.json",
+            channel.name(),
+            subdirs
+                .iter()
+                .copied()
+                .map(Platform::as_str)
+                .sorted()
+                .join("-")
+        )
+    });
+    eprintln!("serializing snapshot to {}", &output_file);
+    let snapshot_path = Path::new(&output_file);
+    if let Some(dir) = snapshot_path.parent() {
+        std::fs::create_dir_all(dir).unwrap();
+    }
+    let snapshot_file = BufWriter::new(std::fs::File::create(snapshot_path).unwrap());
+    serde_json::to_writer(snapshot_file, &snapshot).unwrap();
+}