make rattler-package-streaming compile with wasm (#287)

* make rattler-package-streaming compile with wasm * do not always activate zstd/wasm * add platform::unknown
conda · Aug 24, 2023 · 42ccfdc · 42ccfdc
1 parent 92550d3
commit 42ccfdc
Show file tree

Hide file tree

Showing 10 changed files with 146 additions and 115 deletions.
diff --git a/crates/rattler_conda_types/src/channel/mod.rs b/crates/rattler_conda_types/src/channel/mod.rs
@@ -66,13 +66,20 @@ impl Channel {
             Channel::from_url(url, platforms, config)
         } else if is_path(channel) {
             let path = PathBuf::from(channel);
-            let absolute_path = absolute_path(&path);
-            let url = Url::from_directory_path(absolute_path)
-                .map_err(|_| ParseChannelError::InvalidPath(path))?;
-            Self {
-                platforms,
-                base_url: url,
-                name: Some(channel.to_owned()),
+
+            #[cfg(target_arch = "wasm32")]
+            return Err(ParseChannelError::InvalidPath(path));
+
+            #[cfg(not(target_arch = "wasm32"))]
+            {
+                let absolute_path = absolute_path(&path);
+                let url = Url::from_directory_path(absolute_path)
+                    .map_err(|_| ParseChannelError::InvalidPath(path))?;
+                Self {
+                    platforms,
+                    base_url: url,
+                    name: Some(channel.to_owned()),
+                }
             }
         } else {
             Channel::from_name(channel, platforms, config)

diff --git a/crates/rattler_conda_types/src/match_spec/parse.rs b/crates/rattler_conda_types/src/match_spec/parse.rs
@@ -348,6 +348,9 @@ fn parse(input: &str) -> Result<MatchSpec, ParseMatchSpecError> {
     if is_package_file(input) {
         let _url = match Url::parse(input) {
             Ok(url) => url,
+            #[cfg(target_arch = "wasm32")]
+            Err(_) => return Err(ParseMatchSpecError::InvalidPackagePathOrUrl),
+            #[cfg(not(target_arch = "wasm32"))]
             Err(_) => match PathBuf::from_str(input) {
                 Ok(path) => Url::from_file_path(path)
                     .map_err(|_| ParseMatchSpecError::InvalidPackagePathOrUrl)?,

diff --git a/crates/rattler_conda_types/src/platform.rs b/crates/rattler_conda_types/src/platform.rs
@@ -11,6 +11,7 @@ use thiserror::Error;
 #[derive(EnumIter, Debug, Clone, Copy, Eq, PartialEq, Hash)]
 pub enum Platform {
     NoArch,
+    Unknown,
 
     Linux32,
     Linux64,
@@ -146,7 +147,9 @@ impl Platform {
             target_os = "emscripten",
             windows
         )))]
-        compile_error!("unsupported target os");
+        {
+            return Platform::Unknown;
+        }
     }
 
     /// Returns a string representation of the platform.
@@ -194,7 +197,7 @@ impl Platform {
     /// Return only the platform (linux, win, or osx from the platform enum)
     pub fn only_platform(&self) -> Option<&str> {
         match self {
-            Platform::NoArch => None,
+            Platform::NoArch | Platform::Unknown => None,
             Platform::Linux32
             | Platform::Linux64
             | Platform::LinuxAarch64
@@ -281,6 +284,7 @@ impl From<Platform> for &'static str {
             Platform::Win64 => "win-64",
             Platform::WinArm64 => "win-arm64",
             Platform::Emscripten32 => "emscripten-32",
+            Platform::Unknown => "unknown",
         }
     }
 }
@@ -292,6 +296,7 @@ impl Platform {
     pub fn arch(&self) -> Option<Arch> {
         match self {
             Platform::NoArch => None,
+            Platform::Unknown => None,
             Platform::Linux32 => Some(Arch::X86),
             Platform::Linux64 => Some(Arch::X86_64),
             Platform::LinuxAarch64 => Some(Arch::Aarch64),

diff --git a/crates/rattler_networking/Cargo.toml b/crates/rattler_networking/Cargo.toml
@@ -11,22 +11,27 @@ license.workspace = true
 readme.workspace = true
 
 [features]
+default = ['blocking']
 native-tls = ['reqwest/native-tls']
 rustls-tls = ['reqwest/rustls-tls']
+blocking = ['reqwest/blocking']
 
 [dependencies]
 anyhow = "1.0.71"
 dirs = "5.0.1"
 keyring = "2.0.4"
 lazy_static = "1.4.0"
 libc = "0.2.147"
-reqwest = { version = "0.11.18", features = ["blocking"], default-features = false}
+reqwest = { version = "0.11.18", default-features = false}
 retry-policies = { version = "0.2.0", default-features = false }
 serde = "1.0.171"
 serde_json = "1.0.102"
 thiserror = "1.0.43"
 tracing = "0.1.37"
 
+[target.'cfg( target_arch = "wasm32" )'.dependencies]
+getrandom = { version = "0.2.10", features = ["js"] }
+
 [dev-dependencies]
 anyhow = "1.0.71"
 insta = { version = "1.30.0", features = ["json"] }

diff --git a/crates/rattler_networking/src/lib.rs b/crates/rattler_networking/src/lib.rs
@@ -111,6 +111,7 @@ impl AuthenticatedClient {
     }
 }
 
+#[cfg(feature = "blocking")]
 /// A blocking client that can be used to make authenticated requests, based on the [`reqwest::blocking::Client`]
 pub struct AuthenticatedClientBlocking {
     /// The underlying client
@@ -120,6 +121,7 @@ pub struct AuthenticatedClientBlocking {
     auth_storage: AuthenticationStorage,
 }
 
+#[cfg(feature = "blocking")]
 impl AuthenticatedClientBlocking {
     /// Create a new authenticated client from the given client and authentication storage
     pub fn from_client(
@@ -133,6 +135,7 @@ impl AuthenticatedClientBlocking {
     }
 }
 
+#[cfg(feature = "blocking")]
 impl Default for AuthenticatedClientBlocking {
     fn default() -> Self {
         AuthenticatedClientBlocking {
@@ -142,6 +145,7 @@ impl Default for AuthenticatedClientBlocking {
     }
 }
 
+#[cfg(feature = "blocking")]
 impl AuthenticatedClientBlocking {
     /// Create a GET request builder for the given URL (see also [`reqwest::blocking::Client::get`])
     pub fn get<U: IntoUrl>(&self, url: U) -> reqwest::blocking::RequestBuilder {

diff --git a/crates/rattler_package_streaming/Cargo.toml b/crates/rattler_package_streaming/Cargo.toml
@@ -11,7 +11,7 @@ license.workspace = true
 readme.workspace = true
 
 [dependencies]
-bzip2 = { version = "0.4" }
+bzip2 = "0.4.4"
 chrono = "0.4.26"
 futures-util = { version = "0.3.28", optional = true }
 itertools = "0.11.0"
@@ -24,16 +24,17 @@ tokio = { version = "1", optional = true }
 tokio-util = { version = "0.7", optional = true }
 reqwest = { version = "0.11.18", optional = true, default-features = false }
 url = "2.4.0"
-zip = { version = "0.6.6" }
-zstd = "0.12.3"
+zip = { version = "0.6.6", default-features = false, features = ["deflate", "time"] }
+zstd = { version = "0.12.3", default-features = false }
 rattler_networking = { version = "0.8.0", path = "../rattler_networking", default-features = false }
 
 [features]
-default = ['native-tls']
+default = ["native-tls", "blocking"]
 tokio = ["dep:tokio", "bzip2/tokio", "tokio/fs", "tokio-util/io", "tokio-util/io-util", "reqwest?/stream", "futures-util"]
-native-tls = ['reqwest?/native-tls']
-rustls-tls = ['reqwest?/rustls-tls']
-reqwest = ["reqwest/blocking"]
+native-tls = ["rattler_networking/native-tls"]
+rustls-tls = ["rattler_networking/rustls-tls"]
+blocking = ["rattler_networking/blocking"]
+wasm = ["zstd/wasm"]
 
 [dev-dependencies]
 tempfile = "3.6.0"

diff --git a/crates/rattler_package_streaming/src/reqwest/blocking.rs b/crates/rattler_package_streaming/src/reqwest/blocking.rs
@@ -0,0 +1,98 @@
+//! Functionality to stream and extract packages directly from a [`reqwest::Url`] with a blocking client
+
+use crate::{ExtractError, ExtractResult};
+use rattler_conda_types::package::ArchiveType;
+use rattler_networking::AuthenticatedClientBlocking;
+use reqwest::blocking::Response;
+use reqwest::IntoUrl;
+use std::path::Path;
+
+/// Extracts the contents a `.tar.bz2` package archive from the specified remote location.
+///
+/// ```rust,no_run
+/// # use std::path::Path;
+/// use rattler_package_streaming::reqwest::extract_tar_bz2;
+/// use rattler_networking::AuthenticatedClientBlocking;
+/// # use reqwest::blocking::Client;
+/// let _ = extract_tar_bz2(
+///     AuthenticatedClientBlocking::default(),
+///     "https://conda.anaconda.org/conda-forge/win-64/python-3.11.0-hcf16a7b_0_cpython.tar.bz2",
+///     Path::new("/tmp"))
+///     .unwrap();
+/// ```
+pub fn extract_tar_bz2(
+    client: AuthenticatedClientBlocking,
+    url: impl IntoUrl,
+    destination: &Path,
+) -> Result<ExtractResult, ExtractError> {
+    // Send the request for the file
+    let response = client
+        .get(url)
+        .send()
+        .and_then(Response::error_for_status)
+        .map_err(ExtractError::ReqwestError)?;
+
+    // The `response` is used to stream in the package data
+    crate::read::extract_tar_bz2(response, destination)
+}
+
+/// Extracts the contents a `.conda` package archive from the specified remote location.
+///
+/// ```rust,no_run
+/// # use std::path::Path;
+/// use rattler_package_streaming::reqwest::extract_conda;
+/// use rattler_networking::AuthenticatedClientBlocking;
+/// # use reqwest::blocking::Client;
+/// let _ = extract_conda(
+///     AuthenticatedClientBlocking::default(),
+///     "https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda",
+///     Path::new("/tmp"))
+///     .unwrap();
+/// ```
+pub fn extract_conda(
+    client: AuthenticatedClientBlocking,
+    url: impl IntoUrl,
+    destination: &Path,
+) -> Result<ExtractResult, ExtractError> {
+    // Send the request for the file
+    let response = client
+        .get(url)
+        .send()
+        .and_then(Response::error_for_status)
+        .map_err(ExtractError::ReqwestError)?;
+
+    // The `response` is used to stream in the package data
+    crate::read::extract_conda(response, destination)
+}
+
+/// Extracts the contents a package archive from the specified remote location. The type of package
+/// is determined based on the path of the url.
+///
+/// ```rust,no_run
+/// # use std::path::Path;
+/// use rattler_package_streaming::reqwest::extract;
+/// use rattler_networking::AuthenticatedClientBlocking;
+/// # use reqwest::blocking::Client;
+/// let _ = extract(
+///     AuthenticatedClientBlocking::default(),
+///     "https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda",
+///     Path::new("/tmp"))
+///     .unwrap();
+/// ```
+pub fn extract(
+    client: AuthenticatedClientBlocking,
+    url: impl IntoUrl,
+    destination: &Path,
+) -> Result<ExtractResult, ExtractError> {
+    let url = url
+        .into_url()
+        .map_err(reqwest::Error::from)
+        .map_err(ExtractError::ReqwestError)?;
+
+    match ArchiveType::try_from(Path::new(url.path()))
+        .ok_or(ExtractError::UnsupportedArchiveType)?
+    {
+        ArchiveType::TarBz2 => extract_tar_bz2(client, url, destination),
+        ArchiveType::Conda => extract_conda(client, url, destination),
+    }
+}
diff --git a/crates/rattler_package_streaming/src/reqwest/mod.rs b/crates/rattler_package_streaming/src/reqwest/mod.rs
@@ -3,99 +3,7 @@
 #[cfg(feature = "tokio")]
 pub mod tokio;
 
-use crate::{ExtractError, ExtractResult};
-use rattler_conda_types::package::ArchiveType;
-use rattler_networking::AuthenticatedClientBlocking;
-use reqwest::blocking::Response;
-use reqwest::IntoUrl;
-use std::path::Path;
-
-/// Extracts the contents a `.tar.bz2` package archive from the specified remote location.
-///
-/// ```rust,no_run
-/// # use std::path::Path;
-/// use rattler_package_streaming::reqwest::extract_tar_bz2;
-/// use rattler_networking::AuthenticatedClientBlocking;
-/// # use reqwest::blocking::Client;
-/// let _ = extract_tar_bz2(
-///     AuthenticatedClientBlocking::default(),
-///     "https://conda.anaconda.org/conda-forge/win-64/python-3.11.0-hcf16a7b_0_cpython.tar.bz2",
-///     Path::new("/tmp"))
-///     .unwrap();
-/// ```
-pub fn extract_tar_bz2(
-    client: AuthenticatedClientBlocking,
-    url: impl IntoUrl,
-    destination: &Path,
-) -> Result<ExtractResult, ExtractError> {
-    // Send the request for the file
-    let response = client
-        .get(url)
-        .send()
-        .and_then(Response::error_for_status)
-        .map_err(ExtractError::ReqwestError)?;
-
-    // The `response` is used to stream in the package data
-    crate::read::extract_tar_bz2(response, destination)
-}
-
-/// Extracts the contents a `.conda` package archive from the specified remote location.
-///
-/// ```rust,no_run
-/// # use std::path::Path;
-/// use rattler_package_streaming::reqwest::extract_conda;
-/// use rattler_networking::AuthenticatedClientBlocking;
-/// # use reqwest::blocking::Client;
-/// let _ = extract_conda(
-///     AuthenticatedClientBlocking::default(),
-///     "https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda",
-///     Path::new("/tmp"))
-///     .unwrap();
-/// ```
-pub fn extract_conda(
-    client: AuthenticatedClientBlocking,
-    url: impl IntoUrl,
-    destination: &Path,
-) -> Result<ExtractResult, ExtractError> {
-    // Send the request for the file
-    let response = client
-        .get(url)
-        .send()
-        .and_then(Response::error_for_status)
-        .map_err(ExtractError::ReqwestError)?;
-
-    // The `response` is used to stream in the package data
-    crate::read::extract_conda(response, destination)
-}
-
-/// Extracts the contents a package archive from the specified remote location. The type of package
-/// is determined based on the path of the url.
-///
-/// ```rust,no_run
-/// # use std::path::Path;
-/// use rattler_package_streaming::reqwest::extract;
-/// use rattler_networking::AuthenticatedClientBlocking;
-/// # use reqwest::blocking::Client;
-/// let _ = extract(
-///     AuthenticatedClientBlocking::default(),
-///     "https://conda.anaconda.org/conda-forge/linux-64/python-3.10.8-h4a9ceb5_0_cpython.conda",
-///     Path::new("/tmp"))
-///     .unwrap();
-/// ```
-pub fn extract(
-    client: AuthenticatedClientBlocking,
-    url: impl IntoUrl,
-    destination: &Path,
-) -> Result<ExtractResult, ExtractError> {
-    let url = url
-        .into_url()
-        .map_err(reqwest::Error::from)
-        .map_err(ExtractError::ReqwestError)?;
-
-    match ArchiveType::try_from(Path::new(url.path()))
-        .ok_or(ExtractError::UnsupportedArchiveType)?
-    {
-        ArchiveType::TarBz2 => extract_tar_bz2(client, url, destination),
-        ArchiveType::Conda => extract_conda(client, url, destination),
-    }
-}
+#[cfg(feature = "blocking")]
+pub mod blocking;
+#[cfg(feature = "blocking")]
+pub use blocking::{extract, extract_conda, extract_tar_bz2};
diff --git a/crates/rattler_package_streaming/tests/extract.rs b/crates/rattler_package_streaming/tests/extract.rs
@@ -194,7 +194,7 @@ async fn test_extract_conda_async(#[case] input: &str, #[case] sha256: &str, #[c
     assert_eq!(&format!("{:x}", result.md5), md5);
 }
 
-#[cfg(feature = "reqwest")]
+#[cfg(all(feature = "reqwest", feature = "blocking"))]
 #[apply(url_archives)]
 fn test_extract_url(#[case] url: &str, #[case] sha256: &str, #[case] md5: &str) {
     use rattler_networking::AuthenticatedClientBlocking;

diff --git a/crates/rattler_virtual_packages/src/lib.rs b/crates/rattler_virtual_packages/src/lib.rs
@@ -272,7 +272,7 @@ impl Archspec {
     /// Returns the CPU architecture for the given platform
     pub fn from_platform(platform: Platform) -> Option<Self> {
         let archspec = match platform {
-            Platform::NoArch => return None,
+            Platform::NoArch | Platform::Unknown => return None,
             Platform::Emscripten32 | Platform::Win32 | Platform::Linux32 => "x86",
             Platform::Win64 | Platform::Osx64 | Platform::Linux64 => "x86_64",
             Platform::LinuxAarch64 => "aarch64",