From 58eda9ca8b33b1c83dc1d8fb4bb6c627b00ca778 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Tue, 23 Jul 2024 12:46:58 -0400 Subject: [PATCH 01/32] Adding self-executable --- src/header.sh | 28 ++++++++++++++++++++++++++++ src/main.rs | 6 ++++++ src/pack.rs | 1 + tests/integration_test.rs | 2 ++ 4 files changed, 37 insertions(+) create mode 100644 src/header.sh diff --git a/src/header.sh b/src/header.sh new file mode 100644 index 0000000..1d7e89a --- /dev/null +++ b/src/header.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Self-extracting executable header + +# Function to clean up temporary files +cleanup() { + rm -rf "$TEMP_DIR" +} + +# Set up trap to clean up on exit +trap cleanup EXIT + +# Create a temporary directory +TEMP_DIR=$(mktemp -d) + +# Extract the tarball to the temporary directory +tail -n +$((LINENO + 2)) "$0" | tar xz -C "$TEMP_DIR" + +# Change to the temporary directory +cd "$TEMP_DIR" || exit 1 + +# Execute the main script or binary (adjust as needed) +./main + +# Exit with the status of the main script +exit $? + +# The tarball content will be appended below this line diff --git a/src/main.rs b/src/main.rs index 7344c55..2d9e4fd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -61,6 +61,10 @@ enum Commands { /// This flag allows packing even if PyPI dependencies are present. #[arg(long, default_value = "false")] ignore_pypi_errors: bool, + + /// Create self-extracting executable + #[arg(long, default_value = "false")] + create_executable: bool, }, /// Unpack a pixi environment @@ -103,6 +107,7 @@ async fn main() -> Result<()> { output_file, inject, ignore_pypi_errors, + create_executable, } => { let options = PackOptions { environment, @@ -116,6 +121,7 @@ async fn main() -> Result<()> { }, injected_packages: inject, ignore_pypi_errors, + create_executable, }; tracing::debug!("Running pack command with options: {:?}", options); pack(options).await? diff --git a/src/pack.rs b/src/pack.rs index 4d26498..49bcc5c 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -36,6 +36,7 @@ pub struct PackOptions { pub metadata: PixiPackMetadata, pub injected_packages: Vec, pub ignore_pypi_errors: bool, + pub create_executable: bool, } /// Pack a pixi environment. diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 1b4ee56..c9c902f 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -27,6 +27,7 @@ fn options( #[default(PixiPackMetadata::default())] metadata: PixiPackMetadata, #[default(Some(ShellEnum::Bash(Bash)))] shell: Option, #[default(false)] ignore_pypi_errors: bool, + #[default(false)] create_executable: bool, ) -> Options { let output_dir = tempdir().expect("Couldn't create a temp dir for tests"); let pack_file = output_dir.path().join("environment.tar"); @@ -40,6 +41,7 @@ fn options( metadata, injected_packages: vec![], ignore_pypi_errors, + create_executable, }, unpack_options: UnpackOptions { pack_file, From 175ed05546403d196c3f43f4e3773dbeed84abd1 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Tue, 23 Jul 2024 14:35:26 -0400 Subject: [PATCH 02/32] Added header.sh --- src/header.sh | 90 ++++++++++++++++++++++++++++++++++++++++----------- src/pack.rs | 55 +++++++++++++++++++++++++++---- 2 files changed, 119 insertions(+), 26 deletions(-) diff --git a/src/header.sh b/src/header.sh index 1d7e89a..422bc11 100644 --- a/src/header.sh +++ b/src/header.sh @@ -1,28 +1,80 @@ -#!/bin/bash +#!/bin/sh -# Self-extracting executable header +set -eu -# Function to clean up temporary files -cleanup() { - rm -rf "$TEMP_DIR" -} +INSTALLER_NAME="__NAME__" +INSTALLER_VERSION="__VERSION__" +INSTALLER_PLATFORM="__PLAT__" +PREFIX="__DEFAULT_PREFIX__" +BATCH=0 +FORCE=0 + +USAGE=" +usage: $0 [options] + +Unpacks ${INSTALLER_NAME} ${INSTALLER_VERSION} + +-b run in batch mode (without manual intervention) +-f no error if install prefix already exists +-h print this help message and exit +-p PREFIX install prefix, defaults to $PREFIX +" + +while getopts "bfhp:" x; do + case "$x" in + h) + echo "$USAGE" + exit 2 + ;; + b) + BATCH=1 + ;; + f) + FORCE=1 + ;; + p) + PREFIX="$OPTARG" + ;; + ?) + echo "ERROR: did not recognize option '$x', please try -h" >&2 + exit 1 + ;; + esac +done -# Set up trap to clean up on exit -trap cleanup EXIT +if [ "$FORCE" = "0" ] && [ -e "$PREFIX" ]; then + echo "ERROR: File or directory already exists: '$PREFIX'" >&2 + echo "If you want to update an existing installation, use the -f option." >&2 + exit 1 +fi + +if ! mkdir -p "$PREFIX"; then + echo "ERROR: Could not create directory: '$PREFIX'" >&2 + exit 1 +fi + +PREFIX=$(cd "$PREFIX"; pwd | sed 's@//@/@') +export PREFIX + +echo "PREFIX=$PREFIX" + +extract_range () { + dd if="$0" bs=1 skip="$1" count="$((${2}-${1}))" 2>/dev/null +} -# Create a temporary directory -TEMP_DIR=$(mktemp -d) +last_line=$(grep -anm 1 '^@@END_HEADER@@' "$0" | sed 's/:.*//') +boundary=$(head -n "${last_line}" "$0" | wc -c | sed 's/ //g') -# Extract the tarball to the temporary directory -tail -n +$((LINENO + 2)) "$0" | tar xz -C "$TEMP_DIR" +cd "$PREFIX" -# Change to the temporary directory -cd "$TEMP_DIR" || exit 1 +echo "Unpacking payload ..." +extract_range $boundary | tar -xzf - -# Execute the main script or binary (adjust as needed) -./main +echo "Installation completed." -# Exit with the status of the main script -exit $? +if [ "$BATCH" = "0" ]; then + echo "Thank you for installing ${INSTALLER_NAME}!" +fi -# The tarball content will be appended below this line +exit 0 +@@END_HEADER@@ \ No newline at end of file diff --git a/src/pack.rs b/src/pack.rs index 49bcc5c..a652a98 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -1,7 +1,5 @@ use std::{ - collections::{HashMap, HashSet}, - path::{Path, PathBuf}, - sync::Arc, + collections::{HashMap, HashSet}, path::{Path, PathBuf}, sync::Arc }; use fxhash::FxHashMap; @@ -172,7 +170,8 @@ pub async fn pack(options: PackOptions) -> Result<()> { // Pack = archive the contents. tracing::info!("Creating archive at {}", options.output_file.display()); - archive_directory(output_folder.path(), &options.output_file) + eprintln!("📦 The temp folder contains {:?}", output_folder); + archive_directory(output_folder.path(), &options.output_file, options.create_executable) .await .map_err(|e| anyhow!("could not archive directory: {}", e))?; @@ -244,8 +243,16 @@ async fn download_package( Ok(()) } -/// Archive a directory into a tarball. -async fn archive_directory(input_dir: &Path, archive_target: &Path) -> Result<()> { +async fn archive_directory(input_dir: &Path, archive_target: &Path, create_executable: bool) -> Result<()> { + if create_executable { + eprintln!("📦 Creating self-extracting executable"); + create_self_extracting_executable(input_dir, archive_target).await + } else { + create_tarball(input_dir, archive_target).await + } +} + +async fn create_tarball(input_dir: &Path, archive_target: &Path) -> Result<()> { let outfile = fs::File::create(archive_target).await.map_err(|e| { anyhow!( "could not create archive file at {}: {}", @@ -275,6 +282,40 @@ async fn archive_directory(input_dir: &Path, archive_target: &Path) -> Result<() Ok(()) } +async fn create_self_extracting_executable(input_dir: &Path, archive_target: &Path) -> Result<()> { + let temp_tarball = archive_target.with_extension("tar.gz.tmp"); + create_tarball(input_dir, &temp_tarball).await?; + + let header_path = PathBuf::from("src/header.sh"); + let header = tokio::fs::read_to_string(&header_path).await.map_err(|e| { + anyhow!("could not read header file: {}", e) + })?; + + let executable_path = archive_target.with_extension("sh"); + let mut final_executable = tokio::fs::File::create(executable_path).await.map_err(|e| { + anyhow!("could not create final executable file: {}", e) + })?; + final_executable.write_all(header.as_bytes()).await?; + final_executable.write_all(b"\n").await?; // Add a newline after the header + + let mut tarball = fs::File::open(&temp_tarball).await?; + tokio::io::copy(&mut tarball, &mut final_executable).await?; + + // Make the file executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(archive_target).await?.permissions(); + perms.set_mode(0o755); + fs::set_permissions(archive_target, perms).await?; + } + + // Clean up temporary tarball + fs::remove_file(&temp_tarball).await?; + + Ok(()) +} + /// Create an `environment.yml` file from the given packages. async fn create_environment_file( destination: &Path, @@ -350,4 +391,4 @@ async fn create_repodata_files( } Ok(()) -} +} \ No newline at end of file From 4ae4b35b4d38324410253b20e97088b53fa3af24 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Tue, 23 Jul 2024 23:48:24 -0400 Subject: [PATCH 03/32] Adding archive directly to shar --- src/header.sh | 23 +++++++++++------------ src/pack.rs | 34 +++++++++++++++++++++++----------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/header.sh b/src/header.sh index 422bc11..0a1f9dd 100644 --- a/src/header.sh +++ b/src/header.sh @@ -58,23 +58,22 @@ export PREFIX echo "PREFIX=$PREFIX" -extract_range () { - dd if="$0" bs=1 skip="$1" count="$((${2}-${1}))" 2>/dev/null -} +last_line=$(($(grep -anm 1 '^@@END_HEADER@@' "$0" | sed 's/:.*//') + 1)) -last_line=$(grep -anm 1 '^@@END_HEADER@@' "$0" | sed 's/:.*//') -boundary=$(head -n "${last_line}" "$0" | wc -c | sed 's/ //g') - -cd "$PREFIX" +echo "last_line: $last_line" echo "Unpacking payload ..." -extract_range $boundary | tar -xzf - +tail -n +$last_line "$0" | tar xzv -C "$PREFIX" -echo "Installation completed." +echo "Creating environment using conda" -if [ "$BATCH" = "0" ]; then - echo "Thank you for installing ${INSTALLER_NAME}!" -fi +conda env create -p ./env --file "$PREFIX/environment.yml" + +echo "Environment created" + +# if [ "$BATCH" = "0" ]; then +# echo "Thank you for installing ${INSTALLER_NAME}!" +# fi exit 0 @@END_HEADER@@ \ No newline at end of file diff --git a/src/pack.rs b/src/pack.rs index a652a98..265222c 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -170,7 +170,6 @@ pub async fn pack(options: PackOptions) -> Result<()> { // Pack = archive the contents. tracing::info!("Creating archive at {}", options.output_file.display()); - eprintln!("📦 The temp folder contains {:?}", output_folder); archive_directory(output_folder.path(), &options.output_file, options.create_executable) .await .map_err(|e| anyhow!("could not archive directory: {}", e))?; @@ -283,36 +282,49 @@ async fn create_tarball(input_dir: &Path, archive_target: &Path) -> Result<()> { } async fn create_self_extracting_executable(input_dir: &Path, archive_target: &Path) -> Result<()> { - let temp_tarball = archive_target.with_extension("tar.gz.tmp"); - create_tarball(input_dir, &temp_tarball).await?; + let tarbytes = Vec::new(); + let mut archive = Builder::new(tarbytes); + archive + .append_dir_all(".", input_dir) + .await + .map_err(|e| anyhow!("could not append directory to archive: {}", e))?; + + let mut compressor = archive + .into_inner() + .await + .map_err(|e| anyhow!("could not finish writing archive: {}", e))?; + + compressor + .shutdown() + .await + .map_err(|e| anyhow!("could not flush output: {}", e))?; + let header_path = PathBuf::from("src/header.sh"); let header = tokio::fs::read_to_string(&header_path).await.map_err(|e| { anyhow!("could not read header file: {}", e) })?; let executable_path = archive_target.with_extension("sh"); - let mut final_executable = tokio::fs::File::create(executable_path).await.map_err(|e| { + + let mut final_executable = tokio::fs::File::create(&executable_path).await.map_err(|e| { anyhow!("could not create final executable file: {}", e) })?; + final_executable.write_all(header.as_bytes()).await?; final_executable.write_all(b"\n").await?; // Add a newline after the header - let mut tarball = fs::File::open(&temp_tarball).await?; - tokio::io::copy(&mut tarball, &mut final_executable).await?; + final_executable.write_all(&compressor).await?; // Make the file executable #[cfg(unix)] { use std::os::unix::fs::PermissionsExt; - let mut perms = fs::metadata(archive_target).await?.permissions(); + let mut perms = fs::metadata(&executable_path).await?.permissions(); perms.set_mode(0o755); - fs::set_permissions(archive_target, perms).await?; + fs::set_permissions(&executable_path, perms).await?; } - // Clean up temporary tarball - fs::remove_file(&temp_tarball).await?; - Ok(()) } From e5171a02715d912faea4d80ca24faea97c8c17dd Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Wed, 24 Jul 2024 15:38:39 -0400 Subject: [PATCH 04/32] Finished self-extracting files --- src/header.sh | 175 +++++++++++++++++++++++++++++++++++++++----------- src/main.rs | 25 +++++++- src/pack.rs | 50 +++++++++------ src/unpack.rs | 48 +++++++++++++- 4 files changed, 236 insertions(+), 62 deletions(-) diff --git a/src/header.sh b/src/header.sh index 0a1f9dd..a14c5f8 100644 --- a/src/header.sh +++ b/src/header.sh @@ -2,78 +2,177 @@ set -eu -INSTALLER_NAME="__NAME__" -INSTALLER_VERSION="__VERSION__" -INSTALLER_PLATFORM="__PLAT__" -PREFIX="__DEFAULT_PREFIX__" -BATCH=0 +TEMPDIR=`mktemp -d` +PREFIX="env" FORCE=0 +INSTALLER="conda" # Default to conda +CREATE_ACTIVATION_SCRIPT=false +PARENT_DIR="$(pwd)" USAGE=" usage: $0 [options] -Unpacks ${INSTALLER_NAME} ${INSTALLER_VERSION} +Unpacks an environment packed with pixi-pack --b run in batch mode (without manual intervention) --f no error if install prefix already exists +-f no error if environment already exists -h print this help message and exit --p PREFIX install prefix, defaults to $PREFIX +-p ENV environment prefix, defaults to $PREFIX +-i INSTALLER create the environment using the specified installer defaulting to $INSTALLER +-a create an activation script to activate the environment " -while getopts "bfhp:" x; do - case "$x" in - h) - echo "$USAGE" - exit 2 +create_activation_script() { + local destination="$1" + local prefix="$2" + local shell=$(basename "$3") + + case "$shell" in + bash) + extension="sh" + ;; + zsh) + extension="zsh" ;; - b) - BATCH=1 + fish) + extension="fish" ;; + *) + echo "Unsupported shell: $shell" >&2 + return 1 + ;; + esac + + activate_path="${destination}/activate.${extension}" + + activation_dir="${prefix}/etc/conda/activate.d" + deactivation_dir="${prefix}/etc/conda/deactivate.d" + env_vars_dir="${prefix}/etc/conda/env_vars.d" + state_file="${prefix}/conda-meta/state" + + touch "$activate_path" + echo "export PATH=\"$prefix/bin:\$PATH\"" >> "$activate_path" + echo "export CONDA_PREFIX=\"$prefix\"" >> "$activate_path" + + # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#335 + if [ -d "$activation_dir" ]; then + for file in "${activation_dir}/*"; do + echo ". \"$file\"" >> "$activate_path" + done + fi + + # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#337 + if [ -d "$deactivation_dir" ]; then + for file in "${deactivation_dir}/*"; do + echo ". \"$file\"" >> "$activate_path" + done + fi + + # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#191 + if [ -d "$env_vars_dir" ]; then + declare -A env_vars + + env_var_files=($(find "$env_vars_dir" -type f | sort)) + + for file in "${env_var_files[@]}"; do + if jq empty "$file" 2>/dev/null; then + while IFS="=" read -r key value; do + # Remove quotes from the value + value="${value%\"}" + value="${value#\"}" + env_vars["$key"]="$value" + done < <(jq -r 'to_entries | map("\(.key)=\(.value)") | .[]' "$file") + else + echo "WARNING: Invalid JSON file: $file" >&2 + fi + done + + for key in "${!env_vars[@]}"; do + echo "export $key=\"${env_vars[$key]}\"" >> "$activate_path" + done + fi + + # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#236 + if [ -e "$state_file" ]; then + if ! state_json=$(jq '.' "$state_file" 2>/dev/null); then + echo "WARNING: Invalid JSON in state file: $state_file" >&2 + else + state_env_vars=$(echo "$state_json" | jq -r '.env_vars // {}') + while IFS="=" read -r key value; do + if [ -n "$key" ]; then + if [ -n "${env_vars[$key]}" ]; then + echo "WARNING: environment variable $key already defined in packages (path: $state_file)" >&2 + fi + if [ -n "$value" ]; then + env_vars["${key^^}"]="$value" + else + echo "WARNING: environment variable $key has no string value (path: $state_file)" >&2 + fi + fi + done < <(echo "$state_env_vars" | jq -r 'to_entries | map("\(.key)=\(.value)") | .[]') + fi + echo "export CONDA_ENV_STATE_FILE=\"$state_file\"" >> "$activate_path" + fi + + chmod +x "$activate_path" + echo "Activation script created at $activate_path" +} + +while getopts ":fhai:p:" x; do + case "$x" in f) FORCE=1 ;; p) PREFIX="$OPTARG" ;; - ?) - echo "ERROR: did not recognize option '$x', please try -h" >&2 - exit 1 + i) + INSTALLER="$OPTARG" + ;; + a) + CREATE_ACTIVATION_SCRIPT=true + ;; + h) + echo "$USAGE" + exit 2 ;; esac done -if [ "$FORCE" = "0" ] && [ -e "$PREFIX" ]; then - echo "ERROR: File or directory already exists: '$PREFIX'" >&2 - echo "If you want to update an existing installation, use the -f option." >&2 +if [ "$INSTALLER" != "conda" ] && [ "$INSTALLER" != "micromamba" ]; then + echo "ERROR: Invalid installer: '$INSTALLER'" >&2 exit 1 fi -if ! mkdir -p "$PREFIX"; then - echo "ERROR: Could not create directory: '$PREFIX'" >&2 +if [ "$FORCE" = "0" ] && [ -e "$PREFIX" ]; then + echo "ERROR: File or directory already exists: '$PREFIX'" >&2 + echo "If you want to update an existing environment, use the -f option." >&2 exit 1 +elif [ "$FORCE" = "1" ] && [ -e "$PREFIX" ]; then + rm -rf "$PREFIX" fi -PREFIX=$(cd "$PREFIX"; pwd | sed 's@//@/@') -export PREFIX - -echo "PREFIX=$PREFIX" +PREFIX="$(pwd)/$PREFIX" last_line=$(($(grep -anm 1 '^@@END_HEADER@@' "$0" | sed 's/:.*//') + 1)) -echo "last_line: $last_line" - echo "Unpacking payload ..." -tail -n +$last_line "$0" | tar xzv -C "$PREFIX" +tail -n +$last_line "$0" | tar xzv -C "$TEMPDIR" -echo "Creating environment using conda" +echo "Creating environment using $INSTALLER" -conda env create -p ./env --file "$PREFIX/environment.yml" +cd $TEMPDIR -echo "Environment created" +if [ "$INSTALLER" = "conda" ]; then + conda env create -p $PREFIX --file environment.yml +else + micromamba create -p $PREFIX --file environment.yml +fi + +cd $PARENT_DIR -# if [ "$BATCH" = "0" ]; then -# echo "Thank you for installing ${INSTALLER_NAME}!" -# fi +if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then + create_activation_script "$PARENT_DIR" "$PREFIX" "$SHELL" +fi exit 0 -@@END_HEADER@@ \ No newline at end of file +@@END_HEADER@@ diff --git a/src/main.rs b/src/main.rs index 2d9e4fd..5cb1c07 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,7 +50,7 @@ enum Commands { manifest_path: PathBuf, /// Output file to write the pack to (will be an archive) - #[arg(short, long, default_value = cwd().join("environment.tar").into_os_string())] + #[arg(short, long, default_value = cwd().join("environment").into_os_string())] output_file: PathBuf, /// Inject an additional conda package into the final prefix @@ -63,6 +63,7 @@ enum Commands { ignore_pypi_errors: bool, /// Create self-extracting executable + /// This feature is only available on macOS and Linux. #[arg(long, default_value = "false")] create_executable: bool, }, @@ -109,11 +110,23 @@ async fn main() -> Result<()> { ignore_pypi_errors, create_executable, } => { + if create_executable && is_unsupported_platform(&platform) { + return Err(anyhow::anyhow!("Creating self-extracting executables is only supported on macOS and Linux. Current platform: {}", platform)); + } + + let mut output_file_with_extension = output_file; + + if create_executable { + output_file_with_extension = output_file_with_extension.with_extension("sh"); + } else { + output_file_with_extension = output_file_with_extension.with_extension("tar"); + } + let options = PackOptions { environment, platform, auth_file, - output_file, + output_file: output_file_with_extension, manifest_path, metadata: PixiPackMetadata { version: DEFAULT_PIXI_PACK_VERSION.to_string(), @@ -144,3 +157,11 @@ async fn main() -> Result<()> { Ok(()) } + +/// Check if the given platform supports creating self-extracting executables +fn is_unsupported_platform(platform: &Platform) -> bool { + matches!( + platform, + Platform::Win32 | Platform::Win64 | Platform::WinArm64 + ) +} diff --git a/src/pack.rs b/src/pack.rs index 265222c..485f483 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -1,5 +1,7 @@ use std::{ - collections::{HashMap, HashSet}, path::{Path, PathBuf}, sync::Arc + collections::{HashMap, HashSet}, + path::{Path, PathBuf}, + sync::Arc, }; use fxhash::FxHashMap; @@ -169,10 +171,14 @@ pub async fn pack(options: PackOptions) -> Result<()> { create_environment_file(output_folder.path(), conda_packages.iter().map(|(_, p)| p)).await?; // Pack = archive the contents. - tracing::info!("Creating archive at {}", options.output_file.display()); - archive_directory(output_folder.path(), &options.output_file, options.create_executable) - .await - .map_err(|e| anyhow!("could not archive directory: {}", e))?; + tracing::info!("Creating pack at {}", options.output_file.display()); + archive_directory( + output_folder.path(), + &options.output_file, + options.create_executable, + ) + .await + .map_err(|e| anyhow!("could not archive directory: {}", e))?; let output_size = HumanBytes(get_size(&options.output_file)?).to_string(); tracing::info!( @@ -242,7 +248,11 @@ async fn download_package( Ok(()) } -async fn archive_directory(input_dir: &Path, archive_target: &Path, create_executable: bool) -> Result<()> { +async fn archive_directory( + input_dir: &Path, + archive_target: &Path, + create_executable: bool, +) -> Result<()> { if create_executable { eprintln!("📦 Creating self-extracting executable"); create_self_extracting_executable(input_dir, archive_target).await @@ -281,7 +291,7 @@ async fn create_tarball(input_dir: &Path, archive_target: &Path) -> Result<()> { Ok(()) } -async fn create_self_extracting_executable(input_dir: &Path, archive_target: &Path) -> Result<()> { +async fn create_self_extracting_executable(input_dir: &Path, target: &Path) -> Result<()> { let tarbytes = Vec::new(); let mut archive = Builder::new(tarbytes); @@ -298,20 +308,22 @@ async fn create_self_extracting_executable(input_dir: &Path, archive_target: &Pa compressor .shutdown() .await - .map_err(|e| anyhow!("could not flush output: {}", e))?; - - let header_path = PathBuf::from("src/header.sh"); - let header = tokio::fs::read_to_string(&header_path).await.map_err(|e| { - anyhow!("could not read header file: {}", e) - })?; + .map_err(|e| anyhow!("could not flush output: {}", e))?; + + eprintln!( + "Current directory: {}", + std::env::current_dir().unwrap().display() + ); - let executable_path = archive_target.with_extension("sh"); + const HEADER_CONTENT: &[u8] = include_bytes!("header.sh"); - let mut final_executable = tokio::fs::File::create(&executable_path).await.map_err(|e| { - anyhow!("could not create final executable file: {}", e) - })?; + let executable_path = target.with_extension("sh"); - final_executable.write_all(header.as_bytes()).await?; + let mut final_executable = tokio::fs::File::create(&executable_path) + .await + .map_err(|e| anyhow!("could not create final executable file: {}", e))?; + + final_executable.write_all(HEADER_CONTENT).await?; final_executable.write_all(b"\n").await?; // Add a newline after the header final_executable.write_all(&compressor).await?; @@ -403,4 +415,4 @@ async fn create_repodata_files( } Ok(()) -} \ No newline at end of file +} diff --git a/src/unpack.rs b/src/unpack.rs index c7dbd23..720ea35 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -17,6 +17,7 @@ use rattler_shell::{ shell::{Shell, ShellEnum}, }; use tokio::fs; +use tokio::io::AsyncBufReadExt; use tokio_stream::wrappers::ReadDirStream; use tokio_tar::Archive; use url::Url; @@ -43,9 +44,15 @@ pub async fn unpack(options: UnpackOptions) -> Result<()> { let channel_directory = unpack_dir.join(CHANNEL_DIRECTORY_NAME); tracing::info!("Unarchiving pack to {}", unpack_dir.display()); - unarchive(&options.pack_file, &unpack_dir) - .await - .map_err(|e| anyhow!("Could not unarchive: {}", e))?; + if options.pack_file.extension().unwrap_or_default() == "sh" { + extract_archive_from_shellscript(&options.pack_file, &unpack_dir) + .await + .map_err(|e| anyhow!("Could not extract archive from shell script: {}", e))?; + } else { + unarchive(&options.pack_file, &unpack_dir) + .await + .map_err(|e| anyhow!("Could not unarchive: {}", e))?; + } validate_metadata_file(unpack_dir.join(PIXI_PACK_METADATA_PATH)).await?; @@ -142,6 +149,41 @@ async fn collect_packages(channel_dir: &Path) -> Result Result<()> { + let shell_script = fs::File::open(shell_script_path) + .await + .map_err(|e| anyhow!("could not open shell script: {}", e))?; + + let mut reader = tokio::io::BufReader::new(shell_script); + let mut line = String::new(); + let mut found_end_header = false; + + while reader.read_line(&mut line).await? > 0 { + if line.trim() == "@@END_HEADER@@" { + found_end_header = true; + break; + } + line.clear(); + } + + if !found_end_header { + return Err(anyhow!("Could not find @@END_HEADER@@ in shell script")); + } + + let mut archive = Archive::new(reader); + + archive + .unpack(target_dir) + .await + .map_err(|e| anyhow!("could not unpack archive: {}", e))?; + + Ok(()) +} + /// Unarchive a tarball. pub async fn unarchive(archive_path: &Path, target_dir: &Path) -> Result<()> { let file = fs::File::open(archive_path) From cecfe4effce4227a987868aac069e23461646b36 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Wed, 24 Jul 2024 19:38:00 -0400 Subject: [PATCH 05/32] Adding tests --- .pre-commit-config.yaml | 1 + src/header.sh | 42 ++++++---------- src/pack.rs | 5 -- src/unpack.rs | 7 +-- tests/integration_test.rs | 101 +++++++++++++++++++++++++++++++++++++- 5 files changed, 117 insertions(+), 39 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75753f1..87c28f0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,6 +17,7 @@ repos: types: [text] - id: end-of-file-fixer name: end-of-file-fixer + exclude: src/header.sh entry: pixi run -e lint end-of-file-fixer language: system types: [text] diff --git a/src/header.sh b/src/header.sh index a14c5f8..f0adb14 100644 --- a/src/header.sh +++ b/src/header.sh @@ -7,7 +7,7 @@ PREFIX="env" FORCE=0 INSTALLER="conda" # Default to conda CREATE_ACTIVATION_SCRIPT=false -PARENT_DIR="$(pwd)" +PARENT_DIR="$(dirname "$0")" USAGE=" usage: $0 [options] @@ -27,15 +27,9 @@ create_activation_script() { local shell=$(basename "$3") case "$shell" in - bash) + bash | zsh | fish) extension="sh" ;; - zsh) - extension="zsh" - ;; - fish) - extension="fish" - ;; *) echo "Unsupported shell: $shell" >&2 return 1 @@ -69,26 +63,19 @@ create_activation_script() { # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#191 if [ -d "$env_vars_dir" ]; then - declare -A env_vars - - env_var_files=($(find "$env_vars_dir" -type f | sort)) + env_var_files=$(find "$env_vars_dir" -type f | sort) - for file in "${env_var_files[@]}"; do + for file in $env_var_files; do if jq empty "$file" 2>/dev/null; then - while IFS="=" read -r key value; do + jq -r 'to_entries | map("\(.key)=\(.value)") | .[]' "$file" | while IFS="=" read -r key value; do # Remove quotes from the value - value="${value%\"}" - value="${value#\"}" - env_vars["$key"]="$value" - done < <(jq -r 'to_entries | map("\(.key)=\(.value)") | .[]' "$file") + value=$(echo "$value" | sed 's/^"//; s/"$//') + echo "export $key=\"$value\"" >> "$activate_path" + done else echo "WARNING: Invalid JSON file: $file" >&2 fi done - - for key in "${!env_vars[@]}"; do - echo "export $key=\"${env_vars[$key]}\"" >> "$activate_path" - done fi # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#236 @@ -96,19 +83,18 @@ create_activation_script() { if ! state_json=$(jq '.' "$state_file" 2>/dev/null); then echo "WARNING: Invalid JSON in state file: $state_file" >&2 else - state_env_vars=$(echo "$state_json" | jq -r '.env_vars // {}') - while IFS="=" read -r key value; do + echo "$state_json" | jq -r '.env_vars // {} | to_entries | map("\(.key)=\(.value)") | .[]' | while IFS="=" read -r key value; do if [ -n "$key" ]; then - if [ -n "${env_vars[$key]}" ]; then + if grep -q "export $key=" "$activate_path"; then echo "WARNING: environment variable $key already defined in packages (path: $state_file)" >&2 fi if [ -n "$value" ]; then - env_vars["${key^^}"]="$value" + echo "export ${key}=\"$value\"" >> "$activate_path" else echo "WARNING: environment variable $key has no string value (path: $state_file)" >&2 fi fi - done < <(echo "$state_env_vars" | jq -r 'to_entries | map("\(.key)=\(.value)") | .[]') + done fi echo "export CONDA_ENV_STATE_FILE=\"$state_file\"" >> "$activate_path" fi @@ -151,7 +137,7 @@ elif [ "$FORCE" = "1" ] && [ -e "$PREFIX" ]; then rm -rf "$PREFIX" fi -PREFIX="$(pwd)/$PREFIX" +PREFIX="$PARENT_DIR/$PREFIX" last_line=$(($(grep -anm 1 '^@@END_HEADER@@' "$0" | sed 's/:.*//') + 1)) @@ -175,4 +161,4 @@ if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then fi exit 0 -@@END_HEADER@@ +@@END_HEADER@@ \ No newline at end of file diff --git a/src/pack.rs b/src/pack.rs index 485f483..ffbb0ce 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -310,11 +310,6 @@ async fn create_self_extracting_executable(input_dir: &Path, target: &Path) -> R .await .map_err(|e| anyhow!("could not flush output: {}", e))?; - eprintln!( - "Current directory: {}", - std::env::current_dir().unwrap().display() - ); - const HEADER_CONTENT: &[u8] = include_bytes!("header.sh"); let executable_path = target.with_extension("sh"); diff --git a/src/unpack.rs b/src/unpack.rs index 720ea35..0cd4d27 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -45,7 +45,7 @@ pub async fn unpack(options: UnpackOptions) -> Result<()> { tracing::info!("Unarchiving pack to {}", unpack_dir.display()); if options.pack_file.extension().unwrap_or_default() == "sh" { - extract_archive_from_shellscript(&options.pack_file, &unpack_dir) + unarchive_from_shellscript(&options.pack_file, &unpack_dir) .await .map_err(|e| anyhow!("Could not extract archive from shell script: {}", e))?; } else { @@ -150,10 +150,7 @@ async fn collect_packages(channel_dir: &Path) -> Result Result<()> { +pub async fn unarchive_from_shellscript(shell_script_path: &Path, target_dir: &Path) -> Result<()> { let shell_script = fs::File::open(shell_script_path) .await .map_err(|e| anyhow!("could not open shell script: {}", e))?; diff --git a/tests/integration_test.rs b/tests/integration_test.rs index c9c902f..3be6abc 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -30,7 +30,11 @@ fn options( #[default(false)] create_executable: bool, ) -> Options { let output_dir = tempdir().expect("Couldn't create a temp dir for tests"); - let pack_file = output_dir.path().join("environment.tar"); + let pack_file = if create_executable { + output_dir.path().join("environment.sh") + } else { + output_dir.path().join("environment.tar") + }; Options { pack_options: PackOptions { environment, @@ -270,3 +274,98 @@ async fn test_pypi_ignore( let pack_result = pixi_pack::pack(pack_options).await; assert_eq!(pack_result.is_err(), should_fail); } + +#[rstest] +#[tokio::test] +async fn test_create_executable(options: Options, required_fs_objects: Vec<&'static str>) { + let mut pack_options = options.pack_options; + pack_options.create_executable = true; + pack_options.output_file = options.output_dir.path().join("environment.sh"); + + let pack_file = pack_options.output_file.clone(); + + let pack_result = pixi_pack::pack(pack_options).await; + assert!(pack_result.is_ok(), "{:?}", pack_result); + + assert!(pack_file.exists()); + assert_eq!(pack_file.extension().unwrap(), "sh"); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let metadata = std::fs::metadata(&pack_file).unwrap(); + let permissions = metadata.permissions(); + assert!(permissions.mode() & 0o111 != 0); + } + + let unpack_dir = tempdir().expect("Couldn't create a temp dir for tests"); + let unpack_dir_path = unpack_dir.path(); + let unpack_options = UnpackOptions { + pack_file, + output_directory: unpack_dir_path.to_path_buf(), + shell: Some(ShellEnum::Bash(Bash)), + }; + + let unpack_result = pixi_pack::unpack(unpack_options).await; + assert!(unpack_result.is_ok(), "{:?}", unpack_result); + + let env_dir = unpack_dir_path.join("env"); + assert!(env_dir.exists()); + + let activate_file = unpack_dir_path.join("activate.sh"); + assert!(activate_file.exists()); + + required_fs_objects + .iter() + .map(|dir| env_dir.join(dir)) + .for_each(|dir| { + assert!(dir.exists(), "{:?} does not exist", dir); + }); +} + +#[rstest] +#[tokio::test] +async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<&'static str>) { + let mut pack_options = options.pack_options; + pack_options.create_executable = true; + pack_options.output_file = options.output_dir.path().join("environment.sh"); + + let pack_file = pack_options.output_file.clone(); + + let pack_result = pixi_pack::pack(pack_options).await; + assert!(pack_result.is_ok(), "{:?}", pack_result); + + assert!(pack_file.exists()); + assert_eq!(pack_file.extension().unwrap(), "sh"); + + eprintln!("{:?}", pack_file); + + let output = Command::new("sh") + .arg(pack_file) + .arg("-a") + .output() + .expect("Failed to execute packed file for extraction"); + + assert!(output.status.success(), "Extraction failed: {:?}", output); + + let stdout = String::from_utf8_lossy(&output.stdout); + eprintln!("{:?}", stdout); + + let env_dir = options.output_dir.path().join("env"); + assert!( + env_dir.exists(), + "Environment directory not found after extraction" + ); + let activate_file = options.output_dir.path().join("activate.sh"); + assert!( + activate_file.exists(), + "Activation script not found after extraction" + ); + + required_fs_objects + .iter() + .map(|dir| env_dir.join(dir)) + .for_each(|dir| { + assert!(dir.exists(), "{:?} does not exist", dir); + }); +} From 447a6e2006464d2433e453d366153d5bf2f5d358 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Wed, 24 Jul 2024 21:09:38 -0400 Subject: [PATCH 06/32] Remove extra line in header.sh --- src/header.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/header.sh b/src/header.sh index f0adb14..218297d 100644 --- a/src/header.sh +++ b/src/header.sh @@ -96,7 +96,6 @@ create_activation_script() { fi done fi - echo "export CONDA_ENV_STATE_FILE=\"$state_file\"" >> "$activate_path" fi chmod +x "$activate_path" From c2290a6d5e3f67454ac501f5e4875145d2358da9 Mon Sep 17 00:00:00 2001 From: Pavel Zwerschke Date: Thu, 25 Jul 2024 10:44:29 +0200 Subject: [PATCH 07/32] fix ci --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e75838..b7cba19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -41,9 +41,6 @@ jobs: steps: - name: Checkout branch uses: actions/checkout@v4 - with: - ref: ${{ github.head_ref }} - fetch-depth: 0 - name: Set up pixi uses: prefix-dev/setup-pixi@ba3bb36eb2066252b2363392b7739741bb777659 with: From 506ff30dae7179d5fa76b574868d0e5f1d3206c3 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Thu, 25 Jul 2024 14:46:09 -0400 Subject: [PATCH 08/32] Removing extra print in header file --- tests/integration_test.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 3be6abc..cfa669d 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -338,8 +338,6 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& assert!(pack_file.exists()); assert_eq!(pack_file.extension().unwrap(), "sh"); - eprintln!("{:?}", pack_file); - let output = Command::new("sh") .arg(pack_file) .arg("-a") From 9cf18a3557d523766db45083c46c6d6714c895eb Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Thu, 25 Jul 2024 17:43:13 -0400 Subject: [PATCH 09/32] Fixing extraction error in header.sh --- src/header.sh | 2 +- tests/integration_test.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/header.sh b/src/header.sh index 218297d..0fede3b 100644 --- a/src/header.sh +++ b/src/header.sh @@ -141,7 +141,7 @@ PREFIX="$PARENT_DIR/$PREFIX" last_line=$(($(grep -anm 1 '^@@END_HEADER@@' "$0" | sed 's/:.*//') + 1)) echo "Unpacking payload ..." -tail -n +$last_line "$0" | tar xzv -C "$TEMPDIR" +tail -n +$last_line "$0" | tar -xvf -C "$TEMPDIR" echo "Creating environment using $INSTALLER" diff --git a/tests/integration_test.rs b/tests/integration_test.rs index cfa669d..31cc680 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -325,6 +325,7 @@ async fn test_create_executable(options: Options, required_fs_objects: Vec<&'sta #[rstest] #[tokio::test] +#[cfg(any(target_os = "linux", target_os = "macos"))] async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<&'static str>) { let mut pack_options = options.pack_options; pack_options.create_executable = true; From b23c1a9c330163599a8de4dd6ff5e9f6beff42be Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Sat, 3 Aug 2024 10:53:31 -0400 Subject: [PATCH 10/32] Initialized extractor --- Cargo.lock | 9 +++++++++ Cargo.toml | 3 +++ build.rs | 18 ++++++++++++++++++ extractor/Cargo.toml | 9 +++++++++ extractor/src/main.rs | 17 +++++++++++++++++ src/header.sh | 2 ++ 6 files changed, 58 insertions(+) create mode 100644 build.rs create mode 100644 extractor/Cargo.toml create mode 100644 extractor/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index b12dc81..fbe1b57 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -842,6 +842,15 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "extractor" +version = "0.1.0" +dependencies = [ + "anyhow", + "rattler", + "rattler_shell", +] + [[package]] name = "fastrand" version = "1.9.0" diff --git a/Cargo.toml b/Cargo.toml index 9b53544..ad75be5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,3 +59,6 @@ tempfile = "3.10.1" [dev-dependencies] async-std = "1.12.0" rstest = "0.21.0" + +[workspace] +members = ["extractor"] diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..8cd755a --- /dev/null +++ b/build.rs @@ -0,0 +1,18 @@ +use std::process::Command; + +fn main() { + // Compile the extractor + let status = Command::new("cargo") + .args([ + "build", + "--release", + "--manifest-path", + "extractor/Cargo.toml", + ]) + .status() + .expect("Failed to compile extractor"); + + if !status.success() { + panic!("Failed to compile extractor"); + } +} diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml new file mode 100644 index 0000000..fb2f6b2 --- /dev/null +++ b/extractor/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "extractor" +version = "0.1.0" +edition = "2021" + +[dependencies] +rattler = { version = "0.26.4", default-features = false } +rattler_shell = "0.20.9" +anyhow = "1.*" diff --git a/extractor/src/main.rs b/extractor/src/main.rs new file mode 100644 index 0000000..07e6a4e --- /dev/null +++ b/extractor/src/main.rs @@ -0,0 +1,17 @@ +use anyhow::{anyhow, Result}; +// use rattler::install::Installer; +// use rattler::package_cache::PackageCache; +use std::env; +use std::path::Path; + +fn main() -> Result<()> { + let args: Vec = env::args().collect(); + if args.len() != 3 { + return Err(anyhow!("Usage: {} ", args[0])); + } + + let _archive_dir = Path::new(&args[1]); + let _output_dir = Path::new(&args[2]); + + Ok(()) +} diff --git a/src/header.sh b/src/header.sh index 0fede3b..5fedf4c 100644 --- a/src/header.sh +++ b/src/header.sh @@ -143,6 +143,8 @@ last_line=$(($(grep -anm 1 '^@@END_HEADER@@' "$0" | sed 's/:.*//') + 1)) echo "Unpacking payload ..." tail -n +$last_line "$0" | tar -xvf -C "$TEMPDIR" +tail .... | sh $TEMPDIR + echo "Creating environment using $INSTALLER" cd $TEMPDIR From 4bed8273905db737e74dba2b5dd876fa2f7ea90b Mon Sep 17 00:00:00 2001 From: Parsa Bahraminejad <149900742+prsabahrami@users.noreply.github.com> Date: Sat, 3 Aug 2024 10:56:39 -0400 Subject: [PATCH 11/32] Update chore.yml --- .github/workflows/chore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/chore.yml b/.github/workflows/chore.yml index 475d55b..536315e 100644 --- a/.github/workflows/chore.yml +++ b/.github/workflows/chore.yml @@ -1,7 +1,7 @@ name: Chore on: pull_request: - branches: [main] + branches: [main, selfexec_support] types: [opened, reopened, edited, synchronize] concurrency: From 85d50eba1029b4d92743604c328e5ad77e1b2dd2 Mon Sep 17 00:00:00 2001 From: Parsa Bahraminejad <149900742+prsabahrami@users.noreply.github.com> Date: Sat, 3 Aug 2024 11:06:13 -0400 Subject: [PATCH 12/32] Update ci.yml --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e75838..a715d0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,12 @@ name: CI on: - pull_request: merge_group: + pull_request: + push: + branches: + - main + - selfexec_support + # Automatically stop old builds on the same branch/PR concurrency: From 5990275a5faf53365ea3a16de15f328d6e2e0100 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Sun, 4 Aug 2024 21:01:42 -0400 Subject: [PATCH 13/32] Added extractor --- Cargo.lock | 558 ++++++++++++++++++++++++++++++-------- Cargo.toml | 38 ++- build.rs | 9 +- extractor/Cargo.toml | 24 +- extractor/src/main.rs | 238 +++++++++++++++- src/header.sh | 150 ++++------ src/main.rs | 2 + src/pack.rs | 32 ++- src/unpack.rs | 1 + tests/integration_test.rs | 2 +- 10 files changed, 797 insertions(+), 257 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fbe1b57..9b8e5e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,6 +29,18 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -38,6 +50,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -108,6 +126,15 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "async-broadcast" version = "0.5.1" @@ -520,9 +547,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.4" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" dependencies = [ "clap_builder", "clap_derive", @@ -530,9 +557,9 @@ dependencies = [ [[package]] name = "clap-verbosity-flag" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb9b20c0dd58e4c2e991c8d203bbeb76c11304d1011659686b5b644bc29aa478" +checksum = "63d19864d6b68464c59f7162c9914a0b569ddc2926b4a2d71afe62a9738eff53" dependencies = [ "clap", "log", @@ -540,9 +567,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" dependencies = [ "anstream", "anstyle", @@ -552,9 +579,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck", "proc-macro2", @@ -702,6 +729,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "digest" version = "0.10.7" @@ -734,6 +772,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "either" version = "1.12.0" @@ -794,6 +843,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "erased-serde" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24e2389d65ab4fab27dc2a5de7b191e1f6617d1f1c8855c0dc569c94a4cbb18d" +dependencies = [ + "serde", + "typeid", +] + [[package]] name = "errno" version = "0.3.9" @@ -842,15 +901,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "extractor" -version = "0.1.0" -dependencies = [ - "anyhow", - "rattler", - "rattler_shell", -] - [[package]] name = "fastrand" version = "1.9.0" @@ -868,9 +918,9 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "file_url" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1042c5fdc9f2cf548a139ccd0985fa2460d796f99b08574f72f1f53d179e6591" +checksum = "d0d1df57145d7cda57c95c44a2d64c24f579e2d50b8f4f7a779287293ad3adc0" dependencies = [ "itertools", "percent-encoding", @@ -901,6 +951,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1210,6 +1269,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "halfbrown" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8588661a8607108a5ca69cab034063441a0413a0b041c13618a7dd348021ef6f" +dependencies = [ + "hashbrown 0.14.5", + "serde", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1221,6 +1290,10 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] [[package]] name = "heck" @@ -1401,19 +1474,21 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.26.0" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", "hyper 1.3.1", "hyper-util", - "rustls 0.22.4", + "rustls 0.23.12", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tower-service", + "webpki-roots 0.26.2", ] [[package]] @@ -1573,9 +1648,9 @@ checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" [[package]] name = "itertools" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -1671,6 +1746,70 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.155" @@ -1719,6 +1858,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.21" @@ -1803,13 +1948,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -2212,7 +2358,7 @@ dependencies = [ [[package]] name = "pixi-pack" -version = "0.1.3" +version = "0.1.5" dependencies = [ "anyhow", "async-std", @@ -2223,13 +2369,13 @@ dependencies = [ "indicatif", "rattler", "rattler_conda_types", - "rattler_digest", + "rattler_digest 1.0.0", "rattler_index", "rattler_lock", "rattler_networking", "rattler_package_streaming", "rattler_shell", - "reqwest 0.12.4", + "reqwest 0.12.5", "reqwest-middleware", "rstest", "serde", @@ -2343,6 +2489,52 @@ dependencies = [ "unicase", ] +[[package]] +name = "quinn" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.12", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls 0.23.12", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +dependencies = [ + "libc", + "once_cell", + "socket2 0.5.7", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -2384,18 +2576,15 @@ dependencies = [ [[package]] name = "rattler" -version = "0.26.4" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3d5504e8afc260cceebb79886032ac146c9344c55fbaf9034ca45a0c00b7447" +checksum = "d9ad5c5d7fbab0f7180ff1266c3f63c1e1cefd0774b20f1d89e0820732c60ea9" dependencies = [ "anyhow", - "bytes", - "chrono", "digest", "dirs", "fs-err", "futures", - "fxhash", "humantime", "indexmap 2.2.6", "itertools", @@ -2405,20 +2594,19 @@ dependencies = [ "parking_lot", "rattler_cache", "rattler_conda_types", - "rattler_digest", + "rattler_digest 0.19.5", "rattler_networking", "rattler_package_streaming", "rattler_shell", "reflink-copy", "regex", - "reqwest 0.12.4", + "reqwest 0.12.5", "reqwest-middleware", "simple_spawn_blocking", "smallvec", "tempfile", "thiserror", "tokio", - "tokio-stream", "tracing", "url", "uuid", @@ -2426,22 +2614,21 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.1.0" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdad5b1a62c97fe6acbad6f1421eed77bf75f736a5af44a18f0e46d6d1cd5c81" +checksum = "890a18f5e336f9712991fd9a116c6b074839167b3135348d43d9ae89dffedd2f" dependencies = [ "anyhow", - "chrono", "digest", "dirs", "fxhash", "itertools", "parking_lot", "rattler_conda_types", - "rattler_digest", + "rattler_digest 0.19.5", "rattler_networking", "rattler_package_streaming", - "reqwest 0.12.4", + "reqwest 0.12.5", "reqwest-middleware", "thiserror", "tokio", @@ -2451,26 +2638,30 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.25.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65d6d35c484af9b1a3ce13ace90de388c8a21b1f832bf2ee97b5681a94178326" +checksum = "fa8a3f6f355aebb8a7af60bf5d961adce58689892d646616201d1f5daba1c525" dependencies = [ "chrono", "file_url", "fxhash", "glob", "hex", + "indexmap 2.2.6", "itertools", "lazy-regex", "nom", "purl", - "rattler_digest", + "rattler_digest 0.19.5", "rattler_macros", "regex", "serde", + "serde-untagged", "serde_json", "serde_repr", "serde_with", + "serde_yaml", + "simd-json", "smallvec", "strum", "thiserror", @@ -2481,9 +2672,9 @@ dependencies = [ [[package]] name = "rattler_digest" -version = "0.19.4" +version = "0.19.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf69475918dd44152f7df690b13f2909f34cc762ae18a2e2c55824b546de161" +checksum = "eeb0228f734983274fb6938844123e88aa55158d53ead37e8ae3deb641fe05aa" dependencies = [ "blake2", "digest", @@ -2495,15 +2686,29 @@ dependencies = [ "sha2", ] +[[package]] +name = "rattler_digest" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b747092584fbc2bfef5e4f7286a0b6f81aec1bf67b537ef9f1749cb7931ac7f" +dependencies = [ + "blake2", + "digest", + "hex", + "md-5", + "serde_with", + "sha2", +] + [[package]] name = "rattler_index" -version = "0.19.17" +version = "0.19.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef7d09c64b9710ada881425733f46093124a7d152114cd9aab5f4341179562" +checksum = "fb638cc256c0ccb68f8faf0dad3d703a64d11834323aa746f12e54d8f2bf236e" dependencies = [ "fs-err", "rattler_conda_types", - "rattler_digest", + "rattler_digest 0.19.5", "rattler_package_streaming", "serde_json", "tracing", @@ -2512,9 +2717,9 @@ dependencies = [ [[package]] name = "rattler_lock" -version = "0.22.12" +version = "0.22.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb54f27b97a03b9b2921bd18967947bc5788a8d653fcd6b6bdd18dad192312f" +checksum = "70241633ceba0c8d4396eaaba4c488be0b22e2a9c62639eeafff2841f604fab2" dependencies = [ "chrono", "file_url", @@ -2523,11 +2728,9 @@ dependencies = [ "itertools", "pep440_rs", "pep508_rs", - "purl", "rattler_conda_types", - "rattler_digest", + "rattler_digest 0.19.5", "serde", - "serde_json", "serde_repr", "serde_with", "serde_yaml", @@ -2537,9 +2740,9 @@ dependencies = [ [[package]] name = "rattler_macros" -version = "0.19.3" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10cef20e8356ea6840294e5754c6a8663b0eb1b97f29d517642f0f99215a2483" +checksum = "b4961d74ca0a15a62c83e439dfd9f440f35f8c31dfb71afe990b2d8fbf916f7a" dependencies = [ "quote", "syn 2.0.66", @@ -2547,26 +2750,23 @@ dependencies = [ [[package]] name = "rattler_networking" -version = "0.20.8" +version = "0.20.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c582ad6d82b397d1e1522910b34dc052bbed4dedc0eed7fb640024de0dc6f5f6" +checksum = "0fec041e559f2b4cb21556816f10b3da174932f49280f335b21563d06d2a4737" dependencies = [ "anyhow", "async-trait", "base64 0.22.1", - "bytes", "chrono", "dirs", "fslock", - "futures", "getrandom", "google-cloud-auth", "http 1.1.0", "itertools", "keyring", "netrc-rs", - "pin-project-lite", - "reqwest 0.12.4", + "reqwest 0.12.5", "reqwest-middleware", "retry-policies", "serde", @@ -2578,18 +2778,18 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.21.3" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390c453b80d7904362e121c89f29aee796fb4d38cc7b24fe7ba9e583ef77a27b" +checksum = "c3a65e6c6ebda42aef76886decf080450eb278bb584bebe3e11fb84b3b541b2d" dependencies = [ "bzip2", "chrono", "futures-util", "num_cpus", "rattler_conda_types", - "rattler_digest", + "rattler_digest 0.19.5", "rattler_networking", - "reqwest 0.12.4", + "reqwest 0.12.5", "reqwest-middleware", "serde_json", "tar", @@ -2604,9 +2804,9 @@ dependencies = [ [[package]] name = "rattler_shell" -version = "0.20.9" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17c8a64079dc3a7b8b0070e0d7c4bee4008bf16d799b8b621a9aa88968650c6" +checksum = "3017971018213c53bfdd4679542922269c8e30d235b88b1dd0bc6e25425c3437" dependencies = [ "enum_dispatch", "indexmap 2.2.6", @@ -2657,6 +2857,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ref-cast" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "reflink-copy" version = "0.1.17" @@ -2748,7 +2968,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", @@ -2764,9 +2984,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "async-compression", "base64 0.22.1", @@ -2778,7 +2998,7 @@ dependencies = [ "http-body 1.0.0", "http-body-util", "hyper 1.3.1", - "hyper-rustls 0.26.0", + "hyper-rustls 0.27.2", "hyper-tls 0.6.0", "hyper-util", "ipnet", @@ -2789,18 +3009,19 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.22.4", + "quinn", + "rustls 0.23.12", "rustls-native-certs", "rustls-pemfile 2.1.2", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.1", "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tokio-util", "tower-service", "url", @@ -2814,14 +3035,14 @@ dependencies = [ [[package]] name = "reqwest-middleware" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45d100244a467870f6cb763c4484d010a6bed6bd610b3676e3825d93fb4cfbd" +checksum = "39346a33ddfe6be00cbc17a34ce996818b97b230b87229f10114693becca1268" dependencies = [ "anyhow", "async-trait", "http 1.1.0", - "reqwest 0.12.4", + "reqwest 0.12.5", "serde", "thiserror", "tower-service", @@ -2829,12 +3050,10 @@ dependencies = [ [[package]] name = "retry-policies" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "493b4243e32d6eedd29f9a398896e35c6943a123b55eec97dcaee98310d25810" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" dependencies = [ - "anyhow", - "chrono", "rand", ] @@ -2889,6 +3108,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc_version" version = "0.4.0" @@ -2939,14 +3164,14 @@ dependencies = [ [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ - "log", + "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.6", "subtle", "zeroize", ] @@ -3001,9 +3226,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ "ring", "rustls-pki-types", @@ -3106,18 +3331,29 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] +[[package]] +name = "serde-untagged" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2676ba99bd82f75cae5cbd2c8eda6fa0b8760f18978ea840e980dd5567b5c5b6" +dependencies = [ + "erased-serde", + "serde", + "typeid", +] + [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", @@ -3126,12 +3362,13 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.122" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" dependencies = [ "indexmap 2.2.6", "itoa", + "memchr", "ryu", "serde", ] @@ -3248,6 +3485,34 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd-json" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "570c430b3d902ea083097e853263ae782dfe40857d93db019a12356c8e8143fa" +dependencies = [ + "getrandom", + "halfbrown", + "lexical-core", + "ref-cast", + "serde", + "serde_json", + "simdutf8", + "value-trait", +] + +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "simple_asn1" version = "0.6.2" @@ -3344,9 +3609,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "strum" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" dependencies = [ "strum_macros", ] @@ -3398,6 +3663,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "system-configuration" version = "0.5.1" @@ -3520,27 +3791,26 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "socket2 0.5.7", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -3569,11 +3839,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.22.4", + "rustls 0.23.12", "rustls-pki-types", "tokio", ] @@ -3587,7 +3857,6 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util", ] [[package]] @@ -3742,9 +4011,15 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typed-path" -version = "0.8.0" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04645b6c01cfb2ddabffc7c67ae6bfe7c3e28a5c37d729f6bb498e784f1fd70c" + +[[package]] +name = "typeid" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6069e2cc1d241fd4ff5fa067e8882996fcfce20986d078696e05abccbcf27b43" +checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf" [[package]] name = "typenum" @@ -3819,9 +4094,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -3863,6 +4138,18 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a84c137d37ab0142f0f2ddfe332651fdbf252e7b7dbb4e67b6c1f1b2e925101" +[[package]] +name = "value-trait" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad8db98c1e677797df21ba03fca7d3bf9bec3ca38db930954e4fe6e1ea27eb4" +dependencies = [ + "float-cmp", + "halfbrown", + "itoa", + "ryu", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -4358,6 +4645,26 @@ dependencies = [ "zvariant", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "zeroize" version = "1.8.1" @@ -4366,15 +4673,34 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zip" -version = "0.6.6" +version = "2.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "40dd8c92efc296286ce1fbd16657c5dbefff44f1b4ca01cc5f517d8b7b3d3e2e" dependencies = [ - "byteorder", + "arbitrary", "crc32fast", "crossbeam-utils", + "displaydoc", "flate2", + "indexmap 2.2.6", + "memchr", + "thiserror", "time", + "zopfli", +] + +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ad75be5..582a300 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,9 @@ [package] name = "pixi-pack" description = "A command line tool to pack and unpack conda environments for easy sharing" -version = "0.1.3" +version = "0.1.5" edition = "2021" +build = "build.rs" [features] default = ["native-tls"] @@ -23,28 +24,28 @@ rustls-tls = [ [dependencies] anyhow = "1.*" -clap = { version = "4.5.4", features = ["derive", "string"] } -clap-verbosity-flag = "2.2.0" +clap = { version = "4.5.13", features = ["derive", "string"] } +clap-verbosity-flag = "2.2.1" futures = "0.3.30" indicatif = "0.17.8" -rattler = { version = "0.26.4", default-features = false } -rattler_digest = "0.19.4" -rattler_conda_types = "0.25.2" -rattler_index = "0.19.17" -rattler_lock = "0.22.12" -rattler_networking = { version = "0.20.8", default-features = false } -rattler_package_streaming = { version = "0.21.3", default-features = false } -rattler_shell = "0.20.9" -reqwest = { version = "0.12.4", default-features = false, features = [ +rattler = { version = "0.27.2", default-features = false } +rattler_digest = "1.0.0" +rattler_conda_types = "0.26.3" +rattler_index = "0.19.21" +rattler_lock = "0.22.16" +rattler_networking = { version = "0.20.10", default-features = false } +rattler_package_streaming = { version = "0.21.7", default-features = false } +rattler_shell = "0.21.3" +reqwest = { version = "0.12.5", default-features = false, features = [ "http2", "macos-system-configuration", ] } -reqwest-middleware = "0.3.1" -serde = { version = "1.0.203", features = ["derive"] } -serde_json = "1.0.117" +reqwest-middleware = "0.3.2" +serde = { version = "1.0.204", features = ["derive"] } +serde_json = "1.0.121" serde_yaml = "0.9.34" tokio-tar = "0.3.1" -tokio = { version = "1.37.0", features = ["rt-multi-thread"] } +tokio = { version = "1.39.2", features = ["rt-multi-thread"] } tokio-stream = { version = "0.1.15", features = ["fs"] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = [ @@ -52,13 +53,10 @@ tracing-subscriber = { version = "0.3.18", features = [ "env-filter", ] } tracing-log = "0.2" -url = "2.5.0" +url = "2.5.2" fxhash = "0.2.1" tempfile = "3.10.1" [dev-dependencies] async-std = "1.12.0" rstest = "0.21.0" - -[workspace] -members = ["extractor"] diff --git a/build.rs b/build.rs index 8cd755a..9178de4 100644 --- a/build.rs +++ b/build.rs @@ -1,18 +1,19 @@ use std::process::Command; fn main() { - // Compile the extractor + let extractor_path = "./extractor"; + let status = Command::new("cargo") .args([ "build", "--release", "--manifest-path", - "extractor/Cargo.toml", + &format!("{}/Cargo.toml", extractor_path), ]) .status() - .expect("Failed to compile extractor"); + .expect("Failed to build extractor"); if !status.success() { - panic!("Failed to compile extractor"); + panic!("Failed to compile the extractor project."); } } diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml index fb2f6b2..1e1150f 100644 --- a/extractor/Cargo.toml +++ b/extractor/Cargo.toml @@ -4,6 +4,26 @@ version = "0.1.0" edition = "2021" [dependencies] -rattler = { version = "0.26.4", default-features = false } -rattler_shell = "0.20.9" anyhow = "1.*" +futures = "0.3.30" +rattler = { version = "0.27.2", default-features = false } +rattler_digest = "1.0.0" +rattler_conda_types = "0.26.3" +rattler_index = "0.19.21" +rattler_lock = "0.22.16" +rattler_networking = { version = "0.20.10", default-features = false } +rattler_package_streaming = { version = "0.21.7", default-features = false } +rattler_shell = "0.21.3" +reqwest = { version = "0.12.5", default-features = false, features = [ + "http2", + "macos-system-configuration", +] } +reqwest-middleware = "0.3.2" +serde = { version = "1.0.204", features = ["derive"] } +serde_json = "1.0.121" +serde_yaml = "0.9.34" +tokio = { version = "1.39.2", features = ["rt-multi-thread"] } +tokio-stream = { version = "0.1.15", features = ["fs"] } +url = "2.5.2" +fxhash = "0.2.1" +tempfile = "3.10.1" diff --git a/extractor/src/main.rs b/extractor/src/main.rs index 07e6a4e..adb69b9 100644 --- a/extractor/src/main.rs +++ b/extractor/src/main.rs @@ -1,17 +1,231 @@ -use anyhow::{anyhow, Result}; -// use rattler::install::Installer; -// use rattler::package_cache::PackageCache; -use std::env; -use std::path::Path; - -fn main() -> Result<()> { - let args: Vec = env::args().collect(); - if args.len() != 3 { - return Err(anyhow!("Usage: {} ", args[0])); +use std::path::{Path, PathBuf}; + +use fxhash::FxHashMap; +use tokio::fs; +use tokio_stream::wrappers::ReadDirStream; + +use anyhow::anyhow; +use anyhow::Result; +use futures::{stream, StreamExt, TryStreamExt}; +use rattler::{ + install::Installer, + package_cache::{CacheKey, PackageCache}, +}; +use rattler_conda_types::{PackageRecord, Platform, RepoData, RepoDataRecord}; +use rattler_package_streaming::fs::extract; +use rattler_shell::{ + activation::{ActivationVariables, Activator, PathModificationBehavior}, + shell::{Shell, ShellEnum}, +}; +use serde::{Deserialize, Serialize}; +use url::Url; + +/// The metadata for a "pixi-pack". +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +pub struct PixiPackMetadata { + /// The pack format version. + pub version: String, + /// The platform the pack was created for. + pub platform: Platform, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + if args.len() != 4 { + return Err(anyhow!( + "Usage: {} ", + args[0] + )); + } + + let task = &args[1]; + let input_dir = Path::new(&args[2]); + let output_dir = Path::new(&args[3]); + + if task == "unpack" { + unpack(input_dir, output_dir).await?; + } else if task == "create-script" { + create_activation_script(input_dir, output_dir).await?; + } else { + return Err(anyhow!( + "Unknown task: {}. Task should be either 'unpack' or 'create-script'", + task + )); + } + + Ok(()) +} + +/// Unpack a pixi environment from a directory +pub async fn unpack(archive_dir: &Path, output_dir: &Path) -> Result<()> { + let channel_directory = archive_dir.join(std::env::var("PIXI_PACK_CHANNEL_DIRECTORY").unwrap()); + + validate_metadata_file(archive_dir.join(std::env::var("PIXI_PACK_METADATA_PATH").unwrap())) + .await?; + + create_prefix(&channel_directory, output_dir) + .await + .map_err(|e| anyhow!("Could not create prefix: {}", e))?; + + Ok(()) +} + +async fn collect_packages_in_subdir(subdir: PathBuf) -> Result> { + let repodata = subdir.join("repodata.json"); + + let raw_repodata_json = fs::read_to_string(repodata) + .await + .map_err(|e| anyhow!("could not read repodata in subdir: {}", e))?; + + let repodata: RepoData = serde_json::from_str(&raw_repodata_json).map_err(|e| { + anyhow!( + "could not parse repodata in subdir {}: {}", + subdir.display(), + e + ) + })?; + + let mut conda_packages = repodata.conda_packages; + let packages = repodata.packages; + conda_packages.extend(packages); + Ok(conda_packages) +} + +async fn validate_metadata_file(metadata_file: PathBuf) -> Result<()> { + let metadata_contents = fs::read_to_string(&metadata_file) + .await + .map_err(|e| anyhow!("Could not read metadata file: {}", e))?; + + let metadata: PixiPackMetadata = serde_json::from_str(&metadata_contents)?; + + if metadata.version != std::env::var("PIXI_PACK_DEFAULT_VERSION").unwrap() { + anyhow::bail!("Unsupported pixi-pack version: {}", metadata.version); + } + if metadata.platform != Platform::current() { + anyhow::bail!("The pack was created for a different platform"); } - let _archive_dir = Path::new(&args[1]); - let _output_dir = Path::new(&args[2]); + Ok(()) +} + +/// Collect all packages in a directory. +async fn collect_packages(channel_dir: &Path) -> Result> { + let subdirs = fs::read_dir(channel_dir) + .await + .map_err(|e| anyhow!("could not read channel directory: {}", e))?; + + let stream = ReadDirStream::new(subdirs); + + let packages = stream + .try_filter_map(|entry| async move { + let path = entry.path(); + + if path.is_dir() { + Ok(Some(path)) + } else { + Ok(None) // Ignore non-directory entries + } + }) + .map_ok(collect_packages_in_subdir) + .map_err(|e| anyhow!("could not read channel directory: {}", e)) + .try_buffer_unordered(10) + .try_concat() + .await?; + + Ok(packages) +} + +async fn create_prefix(channel_dir: &Path, target_prefix: &Path) -> Result<()> { + let packages = collect_packages(channel_dir) + .await + .map_err(|e| anyhow!("could not collect packages: {}", e))?; + + let cache_dir = tempfile::tempdir() + .map_err(|e| anyhow!("could not create temporary directory: {}", e))? + .into_path(); + + eprintln!( + "⏳ Extracting and installing {} packages...", + packages.len() + ); + + // extract packages to cache + let package_cache = PackageCache::new(cache_dir); + + let repodata_records: Vec = stream::iter(packages) + .map(|(file_name, package_record)| { + let cache_key = CacheKey::from(&package_record); + + let package_path = channel_dir.join(&package_record.subdir).join(&file_name); + + let url = Url::parse(&format!("file:///{}", file_name)).unwrap(); + + let repodata_record = RepoDataRecord { + package_record, + file_name, + url, + channel: "local".to_string(), + }; + + async { + // We have to prepare the package cache by inserting all packages into it. + // We can only do so by calling `get_or_fetch` on each package, which will + // use the provided closure to fetch the package and insert it into the cache. + package_cache + .get_or_fetch( + cache_key, + |destination| async move { + extract(&package_path, &destination).map(|_| ()) + }, + None, + ) + .await + .map_err(|e| anyhow!("could not extract package: {}", e))?; + + Ok::(repodata_record) + } + }) + .buffer_unordered(50) + .try_collect() + .await?; + + // Invariant: all packages are in the cache + let installer = Installer::default(); + installer + .with_package_cache(package_cache) + .install(&target_prefix, repodata_records) + .await + .map_err(|e| anyhow!("could not install packages: {}", e))?; + + let history_path = target_prefix.join("conda-meta").join("history"); + + fs::write( + history_path, + "// not relevant for pixi but for `conda run -p`", + ) + .await + .map_err(|e| anyhow!("Could not write history file: {}", e))?; + + Ok(()) +} + +async fn create_activation_script(destination: &Path, prefix: &Path) -> Result<()> { + let shell = ShellEnum::default(); + let file_extension = shell.extension(); + let activate_path = destination.join(format!("activate.{}", file_extension)); + let activator = Activator::from_path(prefix, shell, Platform::current())?; + + let result = activator.activation(ActivationVariables { + conda_prefix: None, + path: None, + path_modification_behavior: PathModificationBehavior::Prepend, + })?; + + let contents = result.script.contents()?; + fs::write(activate_path, contents) + .await + .map_err(|e| anyhow!("Could not write activate script: {}", e))?; Ok(()) } diff --git a/src/header.sh b/src/header.sh index 5fedf4c..b28ca27 100644 --- a/src/header.sh +++ b/src/header.sh @@ -5,9 +5,13 @@ set -eu TEMPDIR=`mktemp -d` PREFIX="env" FORCE=0 -INSTALLER="conda" # Default to conda +INSTALLER="rattler" # Default to rattler CREATE_ACTIVATION_SCRIPT=false -PARENT_DIR="$(dirname "$0")" + +# Pixi Constants ./lib.rs +PIXI_PACK_CHANNEL_DIRECTORY="" +PIXI_PACK_METADATA_PATH="" +PIXI_PACK_DEFAULT_VERSION="" USAGE=" usage: $0 [options] @@ -21,87 +25,6 @@ Unpacks an environment packed with pixi-pack -a create an activation script to activate the environment " -create_activation_script() { - local destination="$1" - local prefix="$2" - local shell=$(basename "$3") - - case "$shell" in - bash | zsh | fish) - extension="sh" - ;; - *) - echo "Unsupported shell: $shell" >&2 - return 1 - ;; - esac - - activate_path="${destination}/activate.${extension}" - - activation_dir="${prefix}/etc/conda/activate.d" - deactivation_dir="${prefix}/etc/conda/deactivate.d" - env_vars_dir="${prefix}/etc/conda/env_vars.d" - state_file="${prefix}/conda-meta/state" - - touch "$activate_path" - echo "export PATH=\"$prefix/bin:\$PATH\"" >> "$activate_path" - echo "export CONDA_PREFIX=\"$prefix\"" >> "$activate_path" - - # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#335 - if [ -d "$activation_dir" ]; then - for file in "${activation_dir}/*"; do - echo ". \"$file\"" >> "$activate_path" - done - fi - - # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#337 - if [ -d "$deactivation_dir" ]; then - for file in "${deactivation_dir}/*"; do - echo ". \"$file\"" >> "$activate_path" - done - fi - - # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#191 - if [ -d "$env_vars_dir" ]; then - env_var_files=$(find "$env_vars_dir" -type f | sort) - - for file in $env_var_files; do - if jq empty "$file" 2>/dev/null; then - jq -r 'to_entries | map("\(.key)=\(.value)") | .[]' "$file" | while IFS="=" read -r key value; do - # Remove quotes from the value - value=$(echo "$value" | sed 's/^"//; s/"$//') - echo "export $key=\"$value\"" >> "$activate_path" - done - else - echo "WARNING: Invalid JSON file: $file" >&2 - fi - done - fi - - # https://docs.rs/rattler_shell/latest/src/rattler_shell/activation.rs.html#236 - if [ -e "$state_file" ]; then - if ! state_json=$(jq '.' "$state_file" 2>/dev/null); then - echo "WARNING: Invalid JSON in state file: $state_file" >&2 - else - echo "$state_json" | jq -r '.env_vars // {} | to_entries | map("\(.key)=\(.value)") | .[]' | while IFS="=" read -r key value; do - if [ -n "$key" ]; then - if grep -q "export $key=" "$activate_path"; then - echo "WARNING: environment variable $key already defined in packages (path: $state_file)" >&2 - fi - if [ -n "$value" ]; then - echo "export ${key}=\"$value\"" >> "$activate_path" - else - echo "WARNING: environment variable $key has no string value (path: $state_file)" >&2 - fi - fi - done - fi - fi - - chmod +x "$activate_path" - echo "Activation script created at $activate_path" -} - while getopts ":fhai:p:" x; do case "$x" in f) @@ -123,7 +46,7 @@ while getopts ":fhai:p:" x; do esac done -if [ "$INSTALLER" != "conda" ] && [ "$INSTALLER" != "micromamba" ]; then +if [ "$INSTALLER" != "rattler" ] && [ "$INSTALLER" != "conda" ] && [ "$INSTALLER" != "micromamba" ]; then echo "ERROR: Invalid installer: '$INSTALLER'" >&2 exit 1 fi @@ -132,34 +55,67 @@ if [ "$FORCE" = "0" ] && [ -e "$PREFIX" ]; then echo "ERROR: File or directory already exists: '$PREFIX'" >&2 echo "If you want to update an existing environment, use the -f option." >&2 exit 1 -elif [ "$FORCE" = "1" ] && [ -e "$PREFIX" ]; then +fi + +if [ "$FORCE" = "1" ] && [ -e "$PREFIX" ]; then rm -rf "$PREFIX" fi -PREFIX="$PARENT_DIR/$PREFIX" +if [ "$CREATE_ACTIVATION_SCRIPT" = true ] && [ "$INSTALLER" = "conda" ]; then + echo "ERROR: Activation script creation is only supported with rattler or micromamba as the installer." >&2 + exit 1 +fi -last_line=$(($(grep -anm 1 '^@@END_HEADER@@' "$0" | sed 's/:.*//') + 1)) +mkdir -p "$PREFIX" +PREFIX="$(realpath "$PREFIX")" +PARENT_DIR="$(dirname "$PREFIX")" -echo "Unpacking payload ..." -tail -n +$last_line "$0" | tar -xvf -C "$TEMPDIR" +archive_begin=$(($(grep -anm 1 "^@@END_HEADER@@" "$0" | sed 's/:.*//') + 1)) +archive_end=$(($(grep -anm 1 "^@@END_ARCHIVE@@" "$0" | sed 's/:.*//') - 1)) -tail .... | sh $TEMPDIR +echo "Unpacking payload ..." +tail -n +$archive_begin "$0" | head -n $(($archive_end - $archive_begin + 1)) | tar -xvf - -C "$TEMPDIR" echo "Creating environment using $INSTALLER" -cd $TEMPDIR +if [ "$INSTALLER" = "rattler" ]; then + ( + ls $TEMPDIR + + export PIXI_PACK_CHANNEL_DIRECTORY=$PIXI_PACK_CHANNEL_DIRECTORY + export PIXI_PACK_METADATA_PATH=$PIXI_PACK_METADATA_PATH + export PIXI_PACK_DEFAULT_VERSION=$PIXI_PACK_DEFAULT_VERSION + + rattler_start=$(($archive_end + 2)) -if [ "$INSTALLER" = "conda" ]; then + tail -n +$rattler_start "$0" > "$TEMPDIR/rattler" + chmod +x "$TEMPDIR/rattler" + + "$TEMPDIR/rattler" "unpack" "$TEMPDIR" "$PREFIX" + echo "Environment created at $PREFIX" + + if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then + "$TEMPDIR/rattler" "create-script" "$PARENT_DIR" "$PREFIX" + echo "Activation script created at $PARENT_DIR/activate.sh" + fi + ) +elif [ "$INSTALLER" = "conda" ]; then + cd $TEMPDIR conda env create -p $PREFIX --file environment.yml -else + echo "Environment created at $PREFIX" +elif [ "$INSTALLER" = "micromamba" ]; then + cd $TEMPDIR micromamba create -p $PREFIX --file environment.yml -fi -cd $PARENT_DIR + echo "Environment created at $PREFIX" -if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then - create_activation_script "$PARENT_DIR" "$PREFIX" "$SHELL" + if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then + micromamba shell activate -p $PREFIX > $PARENTDIR/activate.sh + echo "Activation script created at $PARENTDIR/activate.sh" + fi fi +cd $PARENT_DIR + exit 0 @@END_HEADER@@ \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 5cb1c07..6d6a37b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -117,6 +117,8 @@ async fn main() -> Result<()> { let mut output_file_with_extension = output_file; if create_executable { + // TODO: Add support for other platforms + // Change this to shell.extension() output_file_with_extension = output_file_with_extension.with_extension("sh"); } else { output_file_with_extension = output_file_with_extension.with_extension("tar"); diff --git a/src/pack.rs b/src/pack.rs index ffbb0ce..5d788cc 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -21,7 +21,8 @@ use reqwest_middleware::ClientWithMiddleware; use tokio_tar::Builder; use crate::{ - get_size, PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, PIXI_PACK_METADATA_PATH, + get_size, PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, + DEFAULT_PIXI_PACK_VERSION, PIXI_PACK_METADATA_PATH, }; use anyhow::anyhow; @@ -310,18 +311,39 @@ async fn create_self_extracting_executable(input_dir: &Path, target: &Path) -> R .await .map_err(|e| anyhow!("could not flush output: {}", e))?; - const HEADER_CONTENT: &[u8] = include_bytes!("header.sh"); + let header = include_str!("header.sh") + .to_string() + .replace( + "PIXI_PACK_CHANNEL_DIRECTORY=\"\"", + &format!("PIXI_PACK_CHANNEL_DIRECTORY=\"{}\"", CHANNEL_DIRECTORY_NAME), + ) + .replace( + "PIXI_PACK_METADATA_PATH=\"\"", + &format!("PIXI_PACK_METADATA_PATH=\"{}\"", PIXI_PACK_METADATA_PATH), + ) + .replace( + "PIXI_PACK_DEFAULT_VERSION=\"\"", + &format!( + "PIXI_PACK_DEFAULT_VERSION=\"{}\"", + DEFAULT_PIXI_PACK_VERSION + ), + ); let executable_path = target.with_extension("sh"); - let mut final_executable = tokio::fs::File::create(&executable_path) + // Add the binary of extractor to the final executable + const EXTRACTOR: &[u8] = include_bytes!("../extractor/target/release/extractor"); + + let mut final_executable = File::create(&executable_path) .await .map_err(|e| anyhow!("could not create final executable file: {}", e))?; - final_executable.write_all(HEADER_CONTENT).await?; + final_executable.write_all(header.as_bytes()).await?; final_executable.write_all(b"\n").await?; // Add a newline after the header - final_executable.write_all(&compressor).await?; + final_executable.write_all(b"\n").await?; + final_executable.write_all(b"@@END_ARCHIVE@@\n").await?; + final_executable.write_all(EXTRACTOR).await?; // Make the file executable #[cfg(unix)] diff --git a/src/unpack.rs b/src/unpack.rs index 0cd4d27..8cb3a65 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -16,6 +16,7 @@ use rattler_shell::{ activation::{ActivationVariables, Activator, PathModificationBehavior}, shell::{Shell, ShellEnum}, }; + use tokio::fs; use tokio::io::AsyncBufReadExt; use tokio_stream::wrappers::ReadDirStream; diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 31cc680..44a7943 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -341,7 +341,7 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& let output = Command::new("sh") .arg(pack_file) - .arg("-a") + .arg("-af") .output() .expect("Failed to execute packed file for extraction"); From 78352cb6528c261fa88e07620c40c427f07c846a Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Mon, 5 Aug 2024 10:17:42 -0400 Subject: [PATCH 14/32] Fixed unpack shell script --- src/unpack.rs | 42 ++++++++++++++++++++++++--------------- tests/integration_test.rs | 2 ++ 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/src/unpack.rs b/src/unpack.rs index 8cb3a65..642353e 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -1,4 +1,7 @@ -use std::path::{Path, PathBuf}; +use std::{ + io::Cursor, + path::{Path, PathBuf}, +}; use anyhow::{anyhow, Result}; use futures::{ @@ -18,7 +21,7 @@ use rattler_shell::{ }; use tokio::fs; -use tokio::io::AsyncBufReadExt; +use tokio::io::AsyncReadExt; use tokio_stream::wrappers::ReadDirStream; use tokio_tar::Archive; use url::Url; @@ -152,26 +155,33 @@ async fn collect_packages(channel_dir: &Path) -> Result Result<()> { - let shell_script = fs::File::open(shell_script_path) + let mut shell_script = fs::File::open(shell_script_path) .await .map_err(|e| anyhow!("could not open shell script: {}", e))?; - let mut reader = tokio::io::BufReader::new(shell_script); - let mut line = String::new(); - let mut found_end_header = false; + let mut content = Vec::new(); + shell_script + .read_to_end(&mut content) + .await + .map_err(|e| anyhow!("could not read shell script: {}", e))?; - while reader.read_line(&mut line).await? > 0 { - if line.trim() == "@@END_HEADER@@" { - found_end_header = true; - break; - } - line.clear(); - } + let end_header = b"@@END_HEADER@@\n"; + let end_archive = b"@@END_ARCHIVE@@\n"; - if !found_end_header { - return Err(anyhow!("Could not find @@END_HEADER@@ in shell script")); - } + let start = content + .windows(end_header.len()) + .position(|window| window == end_header) + .map(|pos| pos + end_header.len()) + .ok_or_else(|| anyhow!("Could not find @@END_HEADER@@ in shell script"))?; + + let end = content + .windows(end_archive.len()) + .position(|window| window == end_archive) + .ok_or_else(|| anyhow!("Could not find @@END_ARCHIVE@@ in shell script"))?; + + let archive_content = &content[start..end]; + let reader = tokio::io::BufReader::new(Cursor::new(archive_content)); let mut archive = Archive::new(reader); archive diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 44a7943..ef978d0 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -342,6 +342,8 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& let output = Command::new("sh") .arg(pack_file) .arg("-af") + .arg("-p") + .arg(options.output_dir.path().join("env")) .output() .expect("Failed to execute packed file for extraction"); From bab4334a4998c0d055f4e3959efd1e381831ebff Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Wed, 21 Aug 2024 00:33:53 -0400 Subject: [PATCH 15/32] Updated Cargo.toml and Cargo.lock --- Cargo.lock | 804 +++++++++++++++++++++++++---------------------------- Cargo.toml | 6 +- 2 files changed, 385 insertions(+), 425 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9b51db8..61f6a92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aes" version = "0.8.4" @@ -28,18 +34,6 @@ dependencies = [ "cpufeatures", ] -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy 0.7.35", -] - [[package]] name = "ahash" version = "0.8.11" @@ -148,12 +142,14 @@ dependencies = [ [[package]] name = "async-broadcast" -version = "0.5.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c48ccdbf6ca6b121e0f586cbc0e73ae440e56c67c30fa0873b4e110d9c26d2b" +checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" dependencies = [ - "event-listener 2.5.3", + "event-listener 5.3.1", + "event-listener-strategy", "futures-core", + "pin-project-lite", ] [[package]] @@ -207,14 +203,13 @@ dependencies = [ [[package]] name = "async-fs" -version = "1.6.0" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "279cf904654eeebfa37ac9bb1598880884924aab82e290aa65c9e77a0e142e06" +checksum = "ebcd09b382f40fcd159c2d695175b2ae620ffa5f3bd6f664131efff4e8b9e04a" dependencies = [ - "async-lock 2.8.0", - "autocfg", + "async-lock 3.4.0", "blocking", - "futures-lite 1.13.0", + "futures-lite 2.3.0", ] [[package]] @@ -225,7 +220,7 @@ checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ "async-channel 2.3.1", "async-executor", - "async-io 2.3.3", + "async-io 2.3.4", "async-lock 3.4.0", "blocking", "futures-lite 2.3.0", @@ -254,9 +249,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6baa8f0178795da0e71bc42c9e5d13261aac7ee549853162e66a241ba17964" +checksum = "444b0228950ee6501b3568d3c93bf1176a1fdbc3b758dcd9475046d30f4dc7e8" dependencies = [ "async-lock 3.4.0", "cfg-if", @@ -264,11 +259,11 @@ dependencies = [ "futures-io", "futures-lite 2.3.0", "parking", - "polling 3.7.2", + "polling 3.7.3", "rustix 0.38.34", "slab", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -293,19 +288,22 @@ dependencies = [ [[package]] name = "async-process" -version = "1.8.1" +version = "2.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6438ba0a08d81529c69b36700fa2f95837bfe3e776ab39cde9c14d9149da88" +checksum = "a8a07789659a4d385b79b18b9127fc27e1a59e1e89117c78c5ea3b806f016374" dependencies = [ - "async-io 1.13.0", - "async-lock 2.8.0", + "async-channel 2.3.1", + "async-io 2.3.4", + "async-lock 3.4.0", "async-signal", + "async-task", "blocking", "cfg-if", - "event-listener 3.1.0", - "futures-lite 1.13.0", + "event-listener 5.3.1", + "futures-lite 2.3.0", "rustix 0.38.34", - "windows-sys 0.48.0", + "tracing", + "windows-sys 0.59.0", ] [[package]] @@ -316,16 +314,16 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] name = "async-signal" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb3634b73397aa844481f814fad23bbf07fdb0eabec10f2eb95e58944b1ec32" +checksum = "637e00349800c0bdf8bfc21ebbc0b6524abea702b0da4168ac00d070d0c0b9f3" dependencies = [ - "async-io 2.3.3", + "async-io 2.3.4", "async-lock 3.4.0", "atomic-waker", "cfg-if", @@ -334,7 +332,7 @@ dependencies = [ "rustix 0.38.34", "signal-hook-registry", "slab", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -377,7 +375,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -402,7 +400,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] @@ -521,12 +519,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.7" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" +checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -535,6 +534,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.38" @@ -560,9 +565,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.13" +version = "4.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" dependencies = [ "clap_builder", "clap_derive", @@ -580,9 +585,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.13" +version = "4.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" +checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" dependencies = [ "anstream", "anstyle", @@ -599,7 +604,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -648,15 +653,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] @@ -707,7 +712,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -718,7 +723,36 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.72", + "syn 2.0.75", +] + +[[package]] +name = "dbus" +version = "0.9.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bb21987b9fb1613058ba3843121dd18b163b254d8a6e797e144cbac14d96d1b" +dependencies = [ + "libc", + "libdbus-sys", + "winapi", +] + +[[package]] +name = "dbus-secret-service" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1caa0c241c01ad8d99a78d553567d38f873dd3ac16eca33a5370d650ab25584e" +dependencies = [ + "aes", + "block-padding", + "cbc", + "dbus", + "futures-util", + "hkdf", + "num", + "once_cell", + "rand", + "sha2", ] [[package]] @@ -750,7 +784,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -793,7 +827,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -817,6 +851,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "endi" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d8a32ae18130a3c84dd492d4215c3d913c3b07c6b63c2eb3eb7ff1101ab7bf" + [[package]] name = "enum_dispatch" version = "0.3.13" @@ -826,7 +866,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -847,7 +887,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -882,17 +922,6 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" -[[package]] -name = "event-listener" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93877bcde0eb80ca09131a08d23f0a5c18a620b01db137dba666d18cd9b30c2" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - [[package]] name = "event-listener" version = "5.3.1" @@ -931,9 +960,9 @@ checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" [[package]] name = "file_url" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0d1df57145d7cda57c95c44a2d64c24f579e2d50b8f4f7a779287293ad3adc0" +checksum = "5c4602b5420b868957e88e55b927d67e273c764ce9fb6def7369248862928230" dependencies = [ "itertools", "percent-encoding", @@ -944,24 +973,24 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.23" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", - "windows-sys 0.52.0", + "libredox", + "windows-sys 0.59.0", ] [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -1106,7 +1135,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -1256,7 +1285,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.3.0", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -1265,9 +1294,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -1275,7 +1304,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.3.0", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -1466,7 +1495,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.5", + "h2 0.4.6", "http 1.1.0", "http-body 1.0.1", "httparse", @@ -1541,9 +1570,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", "futures-channel", @@ -1611,9 +1640,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -1701,9 +1730,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -1725,16 +1754,16 @@ dependencies = [ [[package]] name = "keyring" -version = "2.3.3" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "363387f0019d714aa60cc30ab4fe501a747f4c08fc58f069dd14be971bd495a0" +checksum = "73b9af47ded4df3067484d7d45758ca2b36bd083bf6d024c2952bbd8af1cdaa4" dependencies = [ "byteorder", - "lazy_static", - "linux-keyutils", + "dbus-secret-service", "secret-service", "security-framework", - "windows-sys 0.52.0", + "windows-sys 0.59.0", + "zbus", ] [[package]] @@ -1766,7 +1795,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -1840,75 +1869,20 @@ dependencies = [ ] [[package]] -name = "lexical-core" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" -dependencies = [ - "lexical-parse-float", - "lexical-parse-integer", - "lexical-util", - "lexical-write-float", - "lexical-write-integer", -] - -[[package]] -name = "lexical-parse-float" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" -dependencies = [ - "lexical-parse-integer", - "lexical-util", - "static_assertions", -] - -[[package]] -name = "lexical-parse-integer" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9" -dependencies = [ - "lexical-util", - "static_assertions", -] - -[[package]] -name = "lexical-util" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" -dependencies = [ - "static_assertions", -] - -[[package]] -name = "lexical-write-float" -version = "0.8.5" +name = "libc" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" -dependencies = [ - "lexical-util", - "lexical-write-integer", - "static_assertions", -] +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] -name = "lexical-write-integer" -version = "0.8.5" +name = "libdbus-sys" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +checksum = "06085512b750d640299b79be4bad3d2fa90a9c00b1fd9e1b46364f66f0485c72" dependencies = [ - "lexical-util", - "static_assertions", + "pkg-config", ] -[[package]] -name = "libc" -version = "0.2.155" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" - [[package]] name = "libredox" version = "0.1.3" @@ -1917,16 +1891,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", -] - -[[package]] -name = "linux-keyutils" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761e49ec5fd8a5a463f9b84e877c373d888935b71c6be78f3767fe2ae6bed18e" -dependencies = [ - "bitflags 2.6.0", - "libc", + "redox_syscall 0.5.3", ] [[package]] @@ -2000,15 +1965,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memoffset" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.9.1" @@ -2039,11 +1995,20 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi 0.3.9", "libc", @@ -2076,14 +2041,15 @@ checksum = "ea2970fbbc8c785e8246234a7bd004ed66cd1ed1a35ec73669a92545e419b836" [[package]] name = "nix" -version = "0.26.4" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", "cfg-if", + "cfg_aliases", "libc", - "memoffset 0.7.1", + "memoffset", ] [[package]] @@ -2203,9 +2169,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "object" -version = "0.36.2" +version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" +checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" dependencies = [ "memchr", ] @@ -2239,7 +2205,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -2386,7 +2352,7 @@ dependencies = [ "phf_shared", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", "unicase", ] @@ -2417,7 +2383,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -2434,9 +2400,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "piper" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1d5c74c9876f070d3e8fd503d748c7d974c3e48da8f41350fa5222ef9b4391" +checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" dependencies = [ "atomic-waker", "fastrand 2.1.0", @@ -2456,13 +2422,13 @@ dependencies = [ "indicatif", "rattler", "rattler_conda_types", - "rattler_digest 1.0.0", + "rattler_digest", "rattler_index", "rattler_lock", "rattler_networking", "rattler_package_streaming", "rattler_shell", - "reqwest 0.12.5", + "reqwest 0.12.7", "reqwest-middleware", "rstest", "serde", @@ -2502,9 +2468,9 @@ dependencies = [ [[package]] name = "polling" -version = "3.7.2" +version = "3.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ed00ed3fbf728b5816498ecd316d1716eecaced9c0c8d2c5a6740ca214985b" +checksum = "cc2790cd301dec6cd3b7a025e4815cf825724a51c98dccfe6a3e55f05ffb6511" dependencies = [ "cfg-if", "concurrent-queue", @@ -2512,7 +2478,7 @@ dependencies = [ "pin-project-lite", "rustix 0.38.34", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2529,21 +2495,11 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy 0.6.6", -] - -[[package]] -name = "proc-macro-crate" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" -dependencies = [ - "once_cell", - "toml_edit 0.19.15", + "zerocopy", ] [[package]] @@ -2552,7 +2508,7 @@ version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" dependencies = [ - "toml_edit 0.21.1", + "toml_edit", ] [[package]] @@ -2581,9 +2537,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" dependencies = [ "bytes", "pin-project-lite", @@ -2591,6 +2547,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.12", + "socket2 0.5.7", "thiserror", "tokio", "tracing", @@ -2598,9 +2555,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" dependencies = [ "bytes", "rand", @@ -2622,6 +2579,7 @@ dependencies = [ "libc", "once_cell", "socket2 0.5.7", + "tracing", "windows-sys 0.52.0", ] @@ -2666,9 +2624,9 @@ dependencies = [ [[package]] name = "rattler" -version = "0.27.2" +version = "0.27.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ad5c5d7fbab0f7180ff1266c3f63c1e1cefd0774b20f1d89e0820732c60ea9" +checksum = "70e94e9ac249e66d45f10216692c29750f39cb95f4ea701c623f0b3138c24fdb" dependencies = [ "anyhow", "digest", @@ -2676,7 +2634,7 @@ dependencies = [ "fs-err", "futures", "humantime", - "indexmap 2.3.0", + "indexmap 2.4.0", "itertools", "memchr", "memmap2", @@ -2684,13 +2642,13 @@ dependencies = [ "parking_lot", "rattler_cache", "rattler_conda_types", - "rattler_digest 0.19.5", + "rattler_digest", "rattler_networking", "rattler_package_streaming", "rattler_shell", "reflink-copy", "regex", - "reqwest 0.12.5", + "reqwest 0.12.7", "reqwest-middleware", "simple_spawn_blocking", "smallvec", @@ -2704,9 +2662,9 @@ dependencies = [ [[package]] name = "rattler_cache" -version = "0.1.4" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "890a18f5e336f9712991fd9a116c6b074839167b3135348d43d9ae89dffedd2f" +checksum = "534344446506ef7b6035b2a3d75937b8375dea7a99da6eeb3631fd478782fa34" dependencies = [ "anyhow", "digest", @@ -2715,10 +2673,10 @@ dependencies = [ "itertools", "parking_lot", "rattler_conda_types", - "rattler_digest 0.19.5", + "rattler_digest", "rattler_networking", "rattler_package_streaming", - "reqwest 0.12.5", + "reqwest 0.12.7", "reqwest-middleware", "thiserror", "tokio", @@ -2728,22 +2686,24 @@ dependencies = [ [[package]] name = "rattler_conda_types" -version = "0.26.3" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8a3f6f355aebb8a7af60bf5d961adce58689892d646616201d1f5daba1c525" +checksum = "d4dc7a58ae237e3f57310095033da58b37d98ce7c167bc928b0f81951279c514" dependencies = [ "chrono", + "dirs", "file_url", "fxhash", "glob", "hex", - "indexmap 2.3.0", + "indexmap 2.4.0", "itertools", "lazy-regex", "nom", "purl", - "rattler_digest 0.19.5", + "rattler_digest", "rattler_macros", + "rattler_redaction", "regex", "serde", "serde-untagged", @@ -2762,9 +2722,9 @@ dependencies = [ [[package]] name = "rattler_digest" -version = "0.19.5" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb0228f734983274fb6938844123e88aa55158d53ead37e8ae3deb641fe05aa" +checksum = "d699fe089907cc0ca66966f6b94e0dc04792a1451b2a60ba49c0abb75fba33da" dependencies = [ "blake2", "digest", @@ -2776,29 +2736,15 @@ dependencies = [ "sha2", ] -[[package]] -name = "rattler_digest" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b747092584fbc2bfef5e4f7286a0b6f81aec1bf67b537ef9f1749cb7931ac7f" -dependencies = [ - "blake2", - "digest", - "hex", - "md-5", - "serde_with", - "sha2", -] - [[package]] name = "rattler_index" -version = "0.19.21" +version = "0.19.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb638cc256c0ccb68f8faf0dad3d703a64d11834323aa746f12e54d8f2bf236e" +checksum = "665dccb14dc21342a87dcb9d095bc1d3b96e76b65e4cfda86d1e65e3b8c06c79" dependencies = [ "fs-err", "rattler_conda_types", - "rattler_digest 0.19.5", + "rattler_digest", "rattler_package_streaming", "serde_json", "tracing", @@ -2807,19 +2753,19 @@ dependencies = [ [[package]] name = "rattler_lock" -version = "0.22.16" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70241633ceba0c8d4396eaaba4c488be0b22e2a9c62639eeafff2841f604fab2" +checksum = "1ce9290c3c0a702806b74b015a5c3dd638211255379f59d261e098ffc12d98d9" dependencies = [ "chrono", "file_url", "fxhash", - "indexmap 2.3.0", + "indexmap 2.4.0", "itertools", "pep440_rs", "pep508_rs", "rattler_conda_types", - "rattler_digest 0.19.5", + "rattler_digest", "serde", "serde_repr", "serde_with", @@ -2830,19 +2776,19 @@ dependencies = [ [[package]] name = "rattler_macros" -version = "0.19.4" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4961d74ca0a15a62c83e439dfd9f440f35f8c31dfb71afe990b2d8fbf916f7a" +checksum = "18011e84fcc8dba03a4af5b70cbb99362968cdace525139df430b4f268ef58e0" dependencies = [ "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] name = "rattler_networking" -version = "0.20.10" +version = "0.21.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fec041e559f2b4cb21556816f10b3da174932f49280f335b21563d06d2a4737" +checksum = "c53c13a325db9d307886a6a31c3c88a3126b006fe618974b528b6dcf1943ece1" dependencies = [ "anyhow", "async-trait", @@ -2856,7 +2802,7 @@ dependencies = [ "itertools", "keyring", "netrc-rs", - "reqwest 0.12.5", + "reqwest 0.12.7", "reqwest-middleware", "retry-policies", "serde", @@ -2868,18 +2814,19 @@ dependencies = [ [[package]] name = "rattler_package_streaming" -version = "0.21.7" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3a65e6c6ebda42aef76886decf080450eb278bb584bebe3e11fb84b3b541b2d" +checksum = "fe4f972fe90d9ebbb055ca3cf3527d9206ff908fee1a39f880d5db209c729976" dependencies = [ "bzip2", "chrono", "futures-util", "num_cpus", "rattler_conda_types", - "rattler_digest 0.19.5", + "rattler_digest", "rattler_networking", - "reqwest 0.12.5", + "rattler_redaction", + "reqwest 0.12.7", "reqwest-middleware", "serde_json", "tar", @@ -2887,19 +2834,31 @@ dependencies = [ "thiserror", "tokio", "tokio-util", + "tracing", "url", "zip", "zstd", ] +[[package]] +name = "rattler_redaction" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14b8739ebf209f017d70f4a27b2726358bade979cc3327b1765163c93a18d46f" +dependencies = [ + "reqwest 0.12.7", + "reqwest-middleware", + "url", +] + [[package]] name = "rattler_shell" -version = "0.21.3" +version = "0.21.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3017971018213c53bfdd4679542922269c8e30d235b88b1dd0bc6e25425c3437" +checksum = "470fb87026522cb8bdf914b7d044644889d62d91e18f57d3e01f90983a4d49b1" dependencies = [ "enum_dispatch", - "indexmap 2.3.0", + "indexmap 2.4.0", "itertools", "rattler_conda_types", "serde_json", @@ -2918,15 +2877,6 @@ dependencies = [ "bitflags 1.3.2", ] -[[package]] -name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - [[package]] name = "redox_syscall" version = "0.5.3" @@ -2938,9 +2888,9 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom", "libredox", @@ -2964,7 +2914,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -2980,9 +2930,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -3059,7 +3009,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", "tokio-native-tls", "tokio-rustls 0.24.1", @@ -3069,21 +3019,21 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots 0.25.4", - "winreg 0.50.0", + "winreg", ] [[package]] name = "reqwest" -version = "0.12.5" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "async-compression", "base64 0.22.1", "bytes", "futures-core", "futures-util", - "h2 0.4.5", + "h2 0.4.6", "http 1.1.0", "http-body 1.0.1", "http-body-util", @@ -3102,13 +3052,13 @@ dependencies = [ "quinn", "rustls 0.23.12", "rustls-native-certs", - "rustls-pemfile 2.1.2", + "rustls-pemfile 2.1.3", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 1.0.1", - "system-configuration", + "system-configuration 0.6.0", "tokio", "tokio-native-tls", "tokio-rustls 0.26.0", @@ -3120,19 +3070,19 @@ dependencies = [ "wasm-streams", "web-sys", "webpki-roots 0.26.3", - "winreg 0.52.0", + "windows-registry", ] [[package]] name = "reqwest-middleware" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39346a33ddfe6be00cbc17a34ce996818b97b230b87229f10114693becca1268" +checksum = "562ceb5a604d3f7c885a792d42c199fd8af239d0a51b2fa6a78aafa092452b04" dependencies = [ "anyhow", "async-trait", "http 1.1.0", - "reqwest 0.12.5", + "reqwest 0.12.7", "serde", "thiserror", "tower-service", @@ -3182,13 +3132,13 @@ checksum = "4165dfae59a39dd41d8dec720d3cbfbc71f69744efb480a3920f5d4e0cc6798d" dependencies = [ "cfg-if", "glob", - "proc-macro-crate 3.1.0", + "proc-macro-crate", "proc-macro2", "quote", "regex", "relative-path", "rustc_version", - "syn 2.0.72", + "syn 2.0.75", "unicode-ident", ] @@ -3200,9 +3150,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "1.1.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" @@ -3268,12 +3218,12 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.2", + "rustls-pemfile 2.1.3", "rustls-pki-types", "schannel", "security-framework", @@ -3290,9 +3240,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ "base64 0.22.1", "rustls-pki-types", @@ -3300,9 +3250,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -3373,9 +3323,9 @@ dependencies = [ [[package]] name = "secret-service" -version = "3.1.0" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5204d39df37f06d1944935232fd2dfe05008def7ca599bf28c0800366c8a8f9" +checksum = "e4d35ad99a181be0a60ffcbe85d680d98f87bdc4d7644ade319b87076b9dbfd4" dependencies = [ "aes", "cbc", @@ -3421,9 +3371,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] @@ -3441,22 +3391,22 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] name = "serde_json" -version = "1.0.122" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ - "indexmap 2.3.0", + "indexmap 2.4.0", "itoa", "memchr", "ryu", @@ -3471,7 +3421,7 @@ checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -3496,7 +3446,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.3.0", + "indexmap 2.4.0", "serde", "serde_derive", "serde_json", @@ -3513,7 +3463,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -3522,7 +3472,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.3.0", + "indexmap 2.4.0", "itoa", "ryu", "serde", @@ -3716,7 +3666,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -3738,9 +3688,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" dependencies = [ "proc-macro2", "quote", @@ -3758,6 +3708,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "system-configuration" @@ -3767,7 +3720,18 @@ checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", "core-foundation", - "system-configuration-sys", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bc6ee10a9b4fcf576e9b0819d95ec16f4d2c02d39fd83ac1c8789785c4a42" +dependencies = [ + "bitflags 2.6.0", + "core-foundation", + "system-configuration-sys 0.6.0", ] [[package]] @@ -3780,6 +3744,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tar" version = "0.4.41" @@ -3793,14 +3767,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.1" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", "fastrand 2.1.0", + "once_cell", "rustix 0.38.34", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3820,7 +3795,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -3881,9 +3856,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", @@ -3904,7 +3879,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -3983,24 +3958,13 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" -[[package]] -name = "toml_edit" -version = "0.19.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" -dependencies = [ - "indexmap 2.3.0", - "toml_datetime", - "winnow", -] - [[package]] name = "toml_edit" version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.3.0", + "indexmap 2.4.0", "toml_datetime", "winnow", ] @@ -4022,15 +3986,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -4051,7 +4015,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -4107,9 +4071,9 @@ checksum = "04645b6c01cfb2ddabffc7c67ae6bfe7c3e28a5c37d729f6bb498e784f1fd70c" [[package]] name = "typeid" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "059d83cc991e7a42fc37bd50941885db0888e34209f8cfd9aab07ddec03bc9cf" +checksum = "0e13db2e0ccd5e14a544e8a246ba2312cd25223f616442d7f2cb0e3db614236e" [[package]] name = "typenum" @@ -4123,7 +4087,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89daebc3e6fd160ac4aa9fc8b3bf71e1f74fbf92367ae71fb83a037e8bf164b9" dependencies = [ - "memoffset 0.9.1", + "memoffset", "tempfile", "winapi", ] @@ -4285,34 +4249,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -4322,9 +4287,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4332,22 +4297,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-streams" @@ -4364,9 +4329,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -4405,11 +4370,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -4458,7 +4423,7 @@ checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -4469,7 +4434,18 @@ checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", +] + +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", ] [[package]] @@ -4509,6 +4485,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -4649,16 +4634,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "xattr" version = "1.3.1" @@ -4672,40 +4647,37 @@ dependencies = [ [[package]] name = "xdg-home" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca91dcf8f93db085f3a0a29358cd0b9d670915468f4290e8b85d118a34211ab8" +checksum = "ec1cdab258fb55c0da61328dc52c8764709b249011b2cad0454c72f0bf10a1f6" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "zbus" -version = "3.15.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "675d170b632a6ad49804c8cf2105d7c31eddd3312555cffd4b740e08e97c25e6" +checksum = "bb97012beadd29e654708a0fdb4c84bc046f537aecfde2c3ee0a9e4b4d48c725" dependencies = [ "async-broadcast", "async-executor", "async-fs", - "async-io 1.13.0", - "async-lock 2.8.0", + "async-io 2.3.4", + "async-lock 3.4.0", "async-process", "async-recursion", "async-task", "async-trait", "blocking", - "byteorder", - "derivative", "enumflags2", - "event-listener 2.5.3", + "event-listener 5.3.1", "futures-core", "futures-sink", "futures-util", "hex", "nix", - "once_cell", "ordered-stream", "rand", "serde", @@ -4714,7 +4686,7 @@ dependencies = [ "static_assertions", "tracing", "uds_windows", - "winapi", + "windows-sys 0.52.0", "xdg-home", "zbus_macros", "zbus_names", @@ -4723,23 +4695,22 @@ dependencies = [ [[package]] name = "zbus_macros" -version = "3.15.2" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7131497b0f887e8061b430c530240063d33bf9455fa34438f388a245da69e0a5" +checksum = "267db9407081e90bbfa46d841d3cbc60f59c0351838c4bc65199ecd79ab1983e" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate", "proc-macro2", "quote", - "regex", - "syn 1.0.109", + "syn 2.0.75", "zvariant_utils", ] [[package]] name = "zbus_names" -version = "2.6.1" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "437d738d3750bed6ca9b8d423ccc7a8eb284f6b1d6d4e225a0e4e6258d864c8d" +checksum = "4b9b1fef7d021261cc16cba64c351d291b715febe0fa10dc3a443ac5a5022e6c" dependencies = [ "serde", "static_assertions", @@ -4752,18 +4723,8 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy-derive" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.72", + "byteorder", + "zerocopy-derive", ] [[package]] @@ -4774,7 +4735,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.75", ] [[package]] @@ -4785,16 +4746,16 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zip" -version = "2.1.6" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40dd8c92efc296286ce1fbd16657c5dbefff44f1b4ca01cc5f517d8b7b3d3e2e" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" dependencies = [ "arbitrary", "crc32fast", "crossbeam-utils", "displaydoc", "flate2", - "indexmap 2.3.0", + "indexmap 2.4.0", "memchr", "thiserror", "time", @@ -4826,18 +4787,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", @@ -4845,13 +4806,12 @@ dependencies = [ [[package]] name = "zvariant" -version = "3.15.2" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eef2be88ba09b358d3b58aca6e41cd853631d44787f319a1383ca83424fb2db" +checksum = "2084290ab9a1c471c38fc524945837734fbf124487e105daec2bb57fd48c81fe" dependencies = [ - "byteorder", + "endi", "enumflags2", - "libc", "serde", "static_assertions", "zvariant_derive", @@ -4859,24 +4819,24 @@ dependencies = [ [[package]] name = "zvariant_derive" -version = "3.15.2" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c24dc0bed72f5f90d1f8bb5b07228cbf63b3c6e9f82d82559d4bae666e7ed9" +checksum = "73e2ba546bda683a90652bac4a279bc146adad1386f25379cf73200d2002c449" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.75", "zvariant_utils", ] [[package]] name = "zvariant_utils" -version = "1.0.1" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7234f0d811589db492d16893e3f21e8e2fd282e6d01b0cddee310322062cc200" +checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.75", ] diff --git a/Cargo.toml b/Cargo.toml index 582a300..a496d0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,11 +30,11 @@ futures = "0.3.30" indicatif = "0.17.8" rattler = { version = "0.27.2", default-features = false } rattler_digest = "1.0.0" -rattler_conda_types = "0.26.3" +rattler_conda_types = "0.27.2" rattler_index = "0.19.21" rattler_lock = "0.22.16" -rattler_networking = { version = "0.20.10", default-features = false } -rattler_package_streaming = { version = "0.21.7", default-features = false } +rattler_networking = { version = "0.21.2", default-features = false } +rattler_package_streaming = { version = "0.22.3", default-features = false } rattler_shell = "0.21.3" reqwest = { version = "0.12.5", default-features = false, features = [ "http2", From 2c70aa1b5fab17959f715f9bd6fd96ebf9cf07e1 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Wed, 21 Aug 2024 00:45:20 -0400 Subject: [PATCH 16/32] Fixed unpack.rs --- Cargo.lock | 36 ++++-------------------------------- src/unpack.rs | 4 ++-- 2 files changed, 6 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7558545..0839074 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2425,8 +2425,8 @@ dependencies = [ "rattler_digest", "rattler_index", "rattler_lock", - "rattler_networking 0.21.1", - "rattler_package_streaming 0.21.7", + "rattler_networking", + "rattler_package_streaming", "rattler_shell", "reqwest 0.12.7", "reqwest-middleware", @@ -2812,34 +2812,6 @@ dependencies = [ "url", ] -[[package]] -name = "rattler_networking" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d635bd42b45a0446ba7cfe21ac07e6efc6016e28ef35c40f6b6e3aa22681667f" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.22.1", - "chrono", - "dirs", - "fslock", - "getrandom", - "google-cloud-auth", - "http 1.1.0", - "itertools", - "keyring", - "netrc-rs", - "reqwest 0.12.5", - "reqwest-middleware", - "retry-policies", - "serde", - "serde_json", - "thiserror", - "tracing", - "url", -] - [[package]] name = "rattler_package_streaming" version = "0.22.3" @@ -2888,7 +2860,7 @@ dependencies = [ "enum_dispatch", "indexmap 2.4.0", "itertools", - "rattler_conda_types 0.27.2", + "rattler_conda_types", "serde_json", "shlex", "tempfile", @@ -3401,7 +3373,7 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" name = "serde" version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2"\ +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] diff --git a/src/unpack.rs b/src/unpack.rs index c333f72..d9e3464 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -47,11 +47,11 @@ pub async fn unpack(options: UnpackOptions) -> Result<()> { tracing::info!("Unarchiving pack to {}", unpack_dir.display()); if options.pack_file.extension().unwrap_or_default() == "sh" { - unarchive_from_shellscript(&options.pack_file, &unpack_dir) + unarchive_from_shellscript(&options.pack_file, unpack_dir) .await .map_err(|e| anyhow!("Could not extract archive from shell script: {}", e))?; } else { - unarchive(&options.pack_file, &unpack_dir) + unarchive(&options.pack_file, unpack_dir) .await .map_err(|e| anyhow!("Could not unarchive: {}", e))?; } From aeb88737df43c6ac8abba26645bb17b878c9ded9 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Thu, 22 Aug 2024 15:58:26 -0400 Subject: [PATCH 17/32] Revert "Merge branch 'main' into selfexec_support" This reverts commit 959188336e280f75de58f26073207c72bd0b0646, reversing changes made to 2c70aa1b5fab17959f715f9bd6fd96ebf9cf07e1. --- .github/workflows/chore.yml | 2 +- .github/workflows/ci.yml | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/chore.yml b/.github/workflows/chore.yml index 536315e..475d55b 100644 --- a/.github/workflows/chore.yml +++ b/.github/workflows/chore.yml @@ -1,7 +1,7 @@ name: Chore on: pull_request: - branches: [main, selfexec_support] + branches: [main] types: [opened, reopened, edited, synchronize] concurrency: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1240ab..b7cba19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,12 +1,7 @@ name: CI on: - merge_group: pull_request: - push: - branches: - - main - - selfexec_support - + merge_group: # Automatically stop old builds on the same branch/PR concurrency: From 0137e5b06ac8da7030bf878a4134fa9a5feadd1e Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Thu, 22 Aug 2024 16:49:24 -0400 Subject: [PATCH 18/32] Fixed cargo.toml and added support for windows --- build.rs | 2 ++ extractor/Cargo.toml | 16 ++++++++-------- src/pack.rs | 3 +++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/build.rs b/build.rs index 9178de4..b698501 100644 --- a/build.rs +++ b/build.rs @@ -9,6 +9,8 @@ fn main() { "--release", "--manifest-path", &format!("{}/Cargo.toml", extractor_path), + "--target-dir", + "extractor/target", ]) .status() .expect("Failed to build extractor"); diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml index 1e1150f..4e94963 100644 --- a/extractor/Cargo.toml +++ b/extractor/Cargo.toml @@ -6,14 +6,14 @@ edition = "2021" [dependencies] anyhow = "1.*" futures = "0.3.30" -rattler = { version = "0.27.2", default-features = false } -rattler_digest = "1.0.0" -rattler_conda_types = "0.26.3" -rattler_index = "0.19.21" -rattler_lock = "0.22.16" -rattler_networking = { version = "0.20.10", default-features = false } -rattler_package_streaming = { version = "0.21.7", default-features = false } -rattler_shell = "0.21.3" +rattler = { version = "0.27.5", default-features = false } +rattler_digest = "1.0.1" +rattler_conda_types = "0.27.2" +rattler_index = "0.19.24" +rattler_lock = "0.22.19" +rattler_networking = { version = "0.21.2", default-features = false } +rattler_package_streaming = { version = "0.22.3", default-features = false } +rattler_shell = "0.21.6" reqwest = { version = "0.12.5", default-features = false, features = [ "http2", "macos-system-configuration", diff --git a/src/pack.rs b/src/pack.rs index 5d788cc..ae17658 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -332,6 +332,9 @@ async fn create_self_extracting_executable(input_dir: &Path, target: &Path) -> R let executable_path = target.with_extension("sh"); // Add the binary of extractor to the final executable + #[cfg(windows)] + const EXTRACTOR: &[u8] = include_bytes!("../extractor/target/release/extractor.exe"); + #[cfg(not(windows))] const EXTRACTOR: &[u8] = include_bytes!("../extractor/target/release/extractor"); let mut final_executable = File::create(&executable_path) From 04b3a457d2adeccf82a0c2915b66783b5d4d244d Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Sat, 31 Aug 2024 13:49:55 -0400 Subject: [PATCH 19/32] Adding tests --- extractor/Cargo.toml | 3 ++ extractor/src/main.rs | 81 +++++++++++++++++++++++++++++++++++++++ pixi.toml | 4 +- tests/integration_test.rs | 5 +-- 4 files changed, 88 insertions(+), 5 deletions(-) diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml index 4e94963..8811653 100644 --- a/extractor/Cargo.toml +++ b/extractor/Cargo.toml @@ -27,3 +27,6 @@ tokio-stream = { version = "0.1.15", features = ["fs"] } url = "2.5.2" fxhash = "0.2.1" tempfile = "3.10.1" + +[dev-dependencies] +rstest = "0.22.0" diff --git a/extractor/src/main.rs b/extractor/src/main.rs index adb69b9..d940bff 100644 --- a/extractor/src/main.rs +++ b/extractor/src/main.rs @@ -229,3 +229,84 @@ async fn create_activation_script(destination: &Path, prefix: &Path) -> Result<( Ok(()) } + +/* --------------------------------------------------------------------------------------------- */ +/* TESTS */ +/* --------------------------------------------------------------------------------------------- */ +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + use serde_json::json; + use std::io::Write; + use tempfile::NamedTempFile; + + fn other_platform() -> Platform { + match Platform::current() { + Platform::Linux64 => Platform::Win64, + _ => Platform::Linux64, + } + } + + #[fixture] + fn metadata_file( + #[default(std::env::var("PIXI_PACK_DEFAULT_VERSION").unwrap())] version: String, + #[default(Platform::current())] platform: Platform, + ) -> NamedTempFile { + let mut metadata_file = NamedTempFile::new().unwrap(); + let metadata = PixiPackMetadata { version, platform }; + let buffer = metadata_file.as_file_mut(); + buffer + .write_all(json!(metadata).to_string().as_bytes()) + .unwrap(); + metadata_file + } + + #[rstest] + #[tokio::test] + async fn test_metadata_file_valid(metadata_file: NamedTempFile) { + assert!(validate_metadata_file(metadata_file.path().to_path_buf()) + .await + .is_ok()) + } + + #[rstest] + #[tokio::test] + async fn test_metadata_file_empty() { + assert!( + validate_metadata_file(NamedTempFile::new().unwrap().path().to_path_buf()) + .await + .is_err() + ) + } + + #[rstest] + #[tokio::test] + async fn test_metadata_file_non_existent() { + assert!(validate_metadata_file(PathBuf::new()).await.is_err()) + } + + #[rstest] + #[tokio::test] + async fn test_metadata_file_invalid_version( + #[with("v0".to_string())] metadata_file: NamedTempFile, + ) { + let result = validate_metadata_file(metadata_file.path().to_path_buf()).await; + let error = result.unwrap_err(); + assert_eq!(error.to_string(), "Unsupported pixi-pack version: v0"); + } + + #[rstest] + #[tokio::test] + async fn test_metadata_file_wrong_platform( + #[with(std::env::var("PIXI_PACK_DEFAULT_VERSION").unwrap(), other_platform())] + metadata_file: NamedTempFile, + ) { + let result = validate_metadata_file(metadata_file.path().to_path_buf()).await; + let error = result.unwrap_err(); + assert_eq!( + error.to_string(), + "The pack was created for a different platform" + ); + } +} diff --git a/pixi.toml b/pixi.toml index 79b8e75..5686dd8 100644 --- a/pixi.toml +++ b/pixi.toml @@ -5,7 +5,9 @@ platforms = ["osx-arm64", "osx-64", "linux-64", "linux-aarch64", "win-64"] [tasks] build = "cargo build --release" -test = "cargo test" +test-pixi-pack = "cargo test" +test-extractor = { cmd = "cargo test --manifest-path extractor/Cargo.toml", env = { PIXI_PACK_DEFAULT_VERSION = "1" } } +test = { depends-on = ["test-pixi-pack", "test-extractor"] } [dependencies] rust = "1.77.2" diff --git a/tests/integration_test.rs b/tests/integration_test.rs index ef978d0..f8f96b3 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -210,7 +210,7 @@ async fn test_compatibility( let pack_file = options.unpack_options.pack_file.clone(); let pack_result = pixi_pack::pack(pack_options).await; - println!("{:?}", pack_result); + assert!(pack_result.is_ok(), "{:?}", pack_result); assert!(pack_file.is_file()); assert!(pack_file.exists()); @@ -349,9 +349,6 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& assert!(output.status.success(), "Extraction failed: {:?}", output); - let stdout = String::from_utf8_lossy(&output.stdout); - eprintln!("{:?}", stdout); - let env_dir = options.output_dir.path().join("env"); assert!( env_dir.exists(), From 62ec962e175e2dfc3f37891e6fa35834c3cae9f5 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Thu, 5 Sep 2024 12:21:16 -0400 Subject: [PATCH 20/32] Fixed merge conflict --- Cargo.lock | 46 +++----------------------------------------- extractor/Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ce75d1..0464c95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -755,35 +755,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "dbus" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bb21987b9fb1613058ba3843121dd18b163b254d8a6e797e144cbac14d96d1b" -dependencies = [ - "libc", - "libdbus-sys", - "winapi", -] - -[[package]] -name = "dbus-secret-service" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1caa0c241c01ad8d99a78d553567d38f873dd3ac16eca33a5370d650ab25584e" -dependencies = [ - "aes", - "block-padding", - "cbc", - "dbus", - "futures-util", - "hkdf", - "num", - "once_cell", - "rand", - "sha2", -] - [[package]] name = "deranged" version = "0.3.11" @@ -4477,17 +4448,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-registry" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" -dependencies = [ - "windows-result", - "windows-strings", - "windows-targets 0.52.6", -] - [[package]] name = "windows-result" version = "0.2.0" @@ -4742,7 +4702,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "zvariant_utils", ] @@ -4866,7 +4826,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "zvariant_utils", ] @@ -4878,5 +4838,5 @@ checksum = "c51bcff7cc3dbb5055396bcf774748c3dab426b4b8659046963523cee4808340" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml index 8811653..a335285 100644 --- a/extractor/Cargo.toml +++ b/extractor/Cargo.toml @@ -10,8 +10,8 @@ rattler = { version = "0.27.5", default-features = false } rattler_digest = "1.0.1" rattler_conda_types = "0.27.2" rattler_index = "0.19.24" -rattler_lock = "0.22.19" -rattler_networking = { version = "0.21.2", default-features = false } +rattler_lock = "0.22.20" +rattler_networking = { version = "0.21.1", default-features = false } rattler_package_streaming = { version = "0.22.3", default-features = false } rattler_shell = "0.21.6" reqwest = { version = "0.12.5", default-features = false, features = [ From b7abd97caf727928c78580a114c55c9dc54f94ca Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Thu, 5 Sep 2024 21:02:03 -0400 Subject: [PATCH 21/32] Update dependencies --- extractor/Cargo.toml | 14 +++++++------- extractor/src/main.rs | 34 +++++++++++++++++++--------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml index a335285..f952b83 100644 --- a/extractor/Cargo.toml +++ b/extractor/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] anyhow = "1.*" futures = "0.3.30" -rattler = { version = "0.27.5", default-features = false } +rattler = { version = "0.27.6", default-features = false } rattler_digest = "1.0.1" rattler_conda_types = "0.27.2" rattler_index = "0.19.24" @@ -14,19 +14,19 @@ rattler_lock = "0.22.20" rattler_networking = { version = "0.21.1", default-features = false } rattler_package_streaming = { version = "0.22.3", default-features = false } rattler_shell = "0.21.6" -reqwest = { version = "0.12.5", default-features = false, features = [ +reqwest = { version = "0.12.7", default-features = false, features = [ "http2", "macos-system-configuration", ] } -reqwest-middleware = "0.3.2" -serde = { version = "1.0.204", features = ["derive"] } -serde_json = "1.0.121" +reqwest-middleware = "0.3.3" +serde = { version = "1.0.209", features = ["derive"] } +serde_json = "1.0.127" serde_yaml = "0.9.34" -tokio = { version = "1.39.2", features = ["rt-multi-thread"] } +tokio = { version = "1.40.0", features = ["rt-multi-thread"] } tokio-stream = { version = "0.1.15", features = ["fs"] } url = "2.5.2" fxhash = "0.2.1" -tempfile = "3.10.1" +tempfile = "3.12.0" [dev-dependencies] rstest = "0.22.0" diff --git a/extractor/src/main.rs b/extractor/src/main.rs index d940bff..dbd28d6 100644 --- a/extractor/src/main.rs +++ b/extractor/src/main.rs @@ -6,7 +6,7 @@ use tokio_stream::wrappers::ReadDirStream; use anyhow::anyhow; use anyhow::Result; -use futures::{stream, StreamExt, TryStreamExt}; +use futures::{stream, StreamExt, TryFutureExt, TryStreamExt}; use rattler::{ install::Installer, package_cache::{CacheKey, PackageCache}, @@ -60,11 +60,12 @@ async fn main() -> Result<()> { /// Unpack a pixi environment from a directory pub async fn unpack(archive_dir: &Path, output_dir: &Path) -> Result<()> { let channel_directory = archive_dir.join(std::env::var("PIXI_PACK_CHANNEL_DIRECTORY").unwrap()); + let cache_dir = archive_dir.join("cache"); validate_metadata_file(archive_dir.join(std::env::var("PIXI_PACK_METADATA_PATH").unwrap())) .await?; - create_prefix(&channel_directory, output_dir) + create_prefix(&channel_directory, output_dir, &cache_dir) .await .map_err(|e| anyhow!("Could not create prefix: {}", e))?; @@ -136,20 +137,16 @@ async fn collect_packages(channel_dir: &Path) -> Result Result<()> { +async fn create_prefix(channel_dir: &Path, target_prefix: &Path, cache_dir: &Path) -> Result<()> { let packages = collect_packages(channel_dir) .await .map_err(|e| anyhow!("could not collect packages: {}", e))?; - let cache_dir = tempfile::tempdir() - .map_err(|e| anyhow!("could not create temporary directory: {}", e))? - .into_path(); - eprintln!( - "⏳ Extracting and installing {} packages...", - packages.len() + "⏳ Extracting and installing {} packages to {}...", + packages.len(), + cache_dir.display() ); - // extract packages to cache let package_cache = PackageCache::new(cache_dir); @@ -175,13 +172,20 @@ async fn create_prefix(channel_dir: &Path, target_prefix: &Path) -> Result<()> { package_cache .get_or_fetch( cache_key, - |destination| async move { - extract(&package_path, &destination).map(|_| ()) + move |destination| { + let package_path_clone = package_path.clone(); + async move { extract(&package_path_clone, &destination).map(|_| ()) } }, None, ) .await - .map_err(|e| anyhow!("could not extract package: {}", e))?; + .map_err(|e| { + anyhow!( + "could not extract \"{}\": {}", + repodata_record.as_ref().name.as_source(), + e + ) + })?; Ok::(repodata_record) } @@ -204,8 +208,8 @@ async fn create_prefix(channel_dir: &Path, target_prefix: &Path) -> Result<()> { history_path, "// not relevant for pixi but for `conda run -p`", ) - .await - .map_err(|e| anyhow!("Could not write history file: {}", e))?; + .map_err(|e| anyhow!("Could not write history file: {}", e)) + .await?; Ok(()) } From 89a242d68b0c07a74970d538fc4a87211ca6a581 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Fri, 20 Sep 2024 01:50:23 -0400 Subject: [PATCH 22/32] added download to pack --- Cargo.toml | 1 - build.rs | 21 -- examples/webserver/my_webserver/__init__.py | 8 - extractor/Cargo.toml | 32 -- extractor/src/main.rs | 316 -------------------- src/header.sh | 63 ++-- src/pack.rs | 72 +++-- src/unpack.rs | 29 +- 8 files changed, 88 insertions(+), 454 deletions(-) delete mode 100644 build.rs delete mode 100644 examples/webserver/my_webserver/__init__.py delete mode 100644 extractor/Cargo.toml delete mode 100644 extractor/src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 54cf706..f974dcb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,6 @@ name = "pixi-pack" description = "A command line tool to pack and unpack conda environments for easy sharing" version = "0.1.8" edition = "2021" -build = "build.rs" [features] default = ["native-tls"] diff --git a/build.rs b/build.rs deleted file mode 100644 index b698501..0000000 --- a/build.rs +++ /dev/null @@ -1,21 +0,0 @@ -use std::process::Command; - -fn main() { - let extractor_path = "./extractor"; - - let status = Command::new("cargo") - .args([ - "build", - "--release", - "--manifest-path", - &format!("{}/Cargo.toml", extractor_path), - "--target-dir", - "extractor/target", - ]) - .status() - .expect("Failed to build extractor"); - - if !status.success() { - panic!("Failed to compile the extractor project."); - } -} diff --git a/examples/webserver/my_webserver/__init__.py b/examples/webserver/my_webserver/__init__.py deleted file mode 100644 index 52f0d71..0000000 --- a/examples/webserver/my_webserver/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/") -def hello(): - return "Hello, World!" diff --git a/extractor/Cargo.toml b/extractor/Cargo.toml deleted file mode 100644 index f952b83..0000000 --- a/extractor/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "extractor" -version = "0.1.0" -edition = "2021" - -[dependencies] -anyhow = "1.*" -futures = "0.3.30" -rattler = { version = "0.27.6", default-features = false } -rattler_digest = "1.0.1" -rattler_conda_types = "0.27.2" -rattler_index = "0.19.24" -rattler_lock = "0.22.20" -rattler_networking = { version = "0.21.1", default-features = false } -rattler_package_streaming = { version = "0.22.3", default-features = false } -rattler_shell = "0.21.6" -reqwest = { version = "0.12.7", default-features = false, features = [ - "http2", - "macos-system-configuration", -] } -reqwest-middleware = "0.3.3" -serde = { version = "1.0.209", features = ["derive"] } -serde_json = "1.0.127" -serde_yaml = "0.9.34" -tokio = { version = "1.40.0", features = ["rt-multi-thread"] } -tokio-stream = { version = "0.1.15", features = ["fs"] } -url = "2.5.2" -fxhash = "0.2.1" -tempfile = "3.12.0" - -[dev-dependencies] -rstest = "0.22.0" diff --git a/extractor/src/main.rs b/extractor/src/main.rs deleted file mode 100644 index dbd28d6..0000000 --- a/extractor/src/main.rs +++ /dev/null @@ -1,316 +0,0 @@ -use std::path::{Path, PathBuf}; - -use fxhash::FxHashMap; -use tokio::fs; -use tokio_stream::wrappers::ReadDirStream; - -use anyhow::anyhow; -use anyhow::Result; -use futures::{stream, StreamExt, TryFutureExt, TryStreamExt}; -use rattler::{ - install::Installer, - package_cache::{CacheKey, PackageCache}, -}; -use rattler_conda_types::{PackageRecord, Platform, RepoData, RepoDataRecord}; -use rattler_package_streaming::fs::extract; -use rattler_shell::{ - activation::{ActivationVariables, Activator, PathModificationBehavior}, - shell::{Shell, ShellEnum}, -}; -use serde::{Deserialize, Serialize}; -use url::Url; - -/// The metadata for a "pixi-pack". -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct PixiPackMetadata { - /// The pack format version. - pub version: String, - /// The platform the pack was created for. - pub platform: Platform, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - if args.len() != 4 { - return Err(anyhow!( - "Usage: {} ", - args[0] - )); - } - - let task = &args[1]; - let input_dir = Path::new(&args[2]); - let output_dir = Path::new(&args[3]); - - if task == "unpack" { - unpack(input_dir, output_dir).await?; - } else if task == "create-script" { - create_activation_script(input_dir, output_dir).await?; - } else { - return Err(anyhow!( - "Unknown task: {}. Task should be either 'unpack' or 'create-script'", - task - )); - } - - Ok(()) -} - -/// Unpack a pixi environment from a directory -pub async fn unpack(archive_dir: &Path, output_dir: &Path) -> Result<()> { - let channel_directory = archive_dir.join(std::env::var("PIXI_PACK_CHANNEL_DIRECTORY").unwrap()); - let cache_dir = archive_dir.join("cache"); - - validate_metadata_file(archive_dir.join(std::env::var("PIXI_PACK_METADATA_PATH").unwrap())) - .await?; - - create_prefix(&channel_directory, output_dir, &cache_dir) - .await - .map_err(|e| anyhow!("Could not create prefix: {}", e))?; - - Ok(()) -} - -async fn collect_packages_in_subdir(subdir: PathBuf) -> Result> { - let repodata = subdir.join("repodata.json"); - - let raw_repodata_json = fs::read_to_string(repodata) - .await - .map_err(|e| anyhow!("could not read repodata in subdir: {}", e))?; - - let repodata: RepoData = serde_json::from_str(&raw_repodata_json).map_err(|e| { - anyhow!( - "could not parse repodata in subdir {}: {}", - subdir.display(), - e - ) - })?; - - let mut conda_packages = repodata.conda_packages; - let packages = repodata.packages; - conda_packages.extend(packages); - Ok(conda_packages) -} - -async fn validate_metadata_file(metadata_file: PathBuf) -> Result<()> { - let metadata_contents = fs::read_to_string(&metadata_file) - .await - .map_err(|e| anyhow!("Could not read metadata file: {}", e))?; - - let metadata: PixiPackMetadata = serde_json::from_str(&metadata_contents)?; - - if metadata.version != std::env::var("PIXI_PACK_DEFAULT_VERSION").unwrap() { - anyhow::bail!("Unsupported pixi-pack version: {}", metadata.version); - } - if metadata.platform != Platform::current() { - anyhow::bail!("The pack was created for a different platform"); - } - - Ok(()) -} - -/// Collect all packages in a directory. -async fn collect_packages(channel_dir: &Path) -> Result> { - let subdirs = fs::read_dir(channel_dir) - .await - .map_err(|e| anyhow!("could not read channel directory: {}", e))?; - - let stream = ReadDirStream::new(subdirs); - - let packages = stream - .try_filter_map(|entry| async move { - let path = entry.path(); - - if path.is_dir() { - Ok(Some(path)) - } else { - Ok(None) // Ignore non-directory entries - } - }) - .map_ok(collect_packages_in_subdir) - .map_err(|e| anyhow!("could not read channel directory: {}", e)) - .try_buffer_unordered(10) - .try_concat() - .await?; - - Ok(packages) -} - -async fn create_prefix(channel_dir: &Path, target_prefix: &Path, cache_dir: &Path) -> Result<()> { - let packages = collect_packages(channel_dir) - .await - .map_err(|e| anyhow!("could not collect packages: {}", e))?; - - eprintln!( - "⏳ Extracting and installing {} packages to {}...", - packages.len(), - cache_dir.display() - ); - // extract packages to cache - let package_cache = PackageCache::new(cache_dir); - - let repodata_records: Vec = stream::iter(packages) - .map(|(file_name, package_record)| { - let cache_key = CacheKey::from(&package_record); - - let package_path = channel_dir.join(&package_record.subdir).join(&file_name); - - let url = Url::parse(&format!("file:///{}", file_name)).unwrap(); - - let repodata_record = RepoDataRecord { - package_record, - file_name, - url, - channel: "local".to_string(), - }; - - async { - // We have to prepare the package cache by inserting all packages into it. - // We can only do so by calling `get_or_fetch` on each package, which will - // use the provided closure to fetch the package and insert it into the cache. - package_cache - .get_or_fetch( - cache_key, - move |destination| { - let package_path_clone = package_path.clone(); - async move { extract(&package_path_clone, &destination).map(|_| ()) } - }, - None, - ) - .await - .map_err(|e| { - anyhow!( - "could not extract \"{}\": {}", - repodata_record.as_ref().name.as_source(), - e - ) - })?; - - Ok::(repodata_record) - } - }) - .buffer_unordered(50) - .try_collect() - .await?; - - // Invariant: all packages are in the cache - let installer = Installer::default(); - installer - .with_package_cache(package_cache) - .install(&target_prefix, repodata_records) - .await - .map_err(|e| anyhow!("could not install packages: {}", e))?; - - let history_path = target_prefix.join("conda-meta").join("history"); - - fs::write( - history_path, - "// not relevant for pixi but for `conda run -p`", - ) - .map_err(|e| anyhow!("Could not write history file: {}", e)) - .await?; - - Ok(()) -} - -async fn create_activation_script(destination: &Path, prefix: &Path) -> Result<()> { - let shell = ShellEnum::default(); - let file_extension = shell.extension(); - let activate_path = destination.join(format!("activate.{}", file_extension)); - let activator = Activator::from_path(prefix, shell, Platform::current())?; - - let result = activator.activation(ActivationVariables { - conda_prefix: None, - path: None, - path_modification_behavior: PathModificationBehavior::Prepend, - })?; - - let contents = result.script.contents()?; - fs::write(activate_path, contents) - .await - .map_err(|e| anyhow!("Could not write activate script: {}", e))?; - - Ok(()) -} - -/* --------------------------------------------------------------------------------------------- */ -/* TESTS */ -/* --------------------------------------------------------------------------------------------- */ -#[cfg(test)] -mod tests { - use super::*; - use rstest::*; - use serde_json::json; - use std::io::Write; - use tempfile::NamedTempFile; - - fn other_platform() -> Platform { - match Platform::current() { - Platform::Linux64 => Platform::Win64, - _ => Platform::Linux64, - } - } - - #[fixture] - fn metadata_file( - #[default(std::env::var("PIXI_PACK_DEFAULT_VERSION").unwrap())] version: String, - #[default(Platform::current())] platform: Platform, - ) -> NamedTempFile { - let mut metadata_file = NamedTempFile::new().unwrap(); - let metadata = PixiPackMetadata { version, platform }; - let buffer = metadata_file.as_file_mut(); - buffer - .write_all(json!(metadata).to_string().as_bytes()) - .unwrap(); - metadata_file - } - - #[rstest] - #[tokio::test] - async fn test_metadata_file_valid(metadata_file: NamedTempFile) { - assert!(validate_metadata_file(metadata_file.path().to_path_buf()) - .await - .is_ok()) - } - - #[rstest] - #[tokio::test] - async fn test_metadata_file_empty() { - assert!( - validate_metadata_file(NamedTempFile::new().unwrap().path().to_path_buf()) - .await - .is_err() - ) - } - - #[rstest] - #[tokio::test] - async fn test_metadata_file_non_existent() { - assert!(validate_metadata_file(PathBuf::new()).await.is_err()) - } - - #[rstest] - #[tokio::test] - async fn test_metadata_file_invalid_version( - #[with("v0".to_string())] metadata_file: NamedTempFile, - ) { - let result = validate_metadata_file(metadata_file.path().to_path_buf()).await; - let error = result.unwrap_err(); - assert_eq!(error.to_string(), "Unsupported pixi-pack version: v0"); - } - - #[rstest] - #[tokio::test] - async fn test_metadata_file_wrong_platform( - #[with(std::env::var("PIXI_PACK_DEFAULT_VERSION").unwrap(), other_platform())] - metadata_file: NamedTempFile, - ) { - let result = validate_metadata_file(metadata_file.path().to_path_buf()).await; - let error = result.unwrap_err(); - assert_eq!( - error.to_string(), - "The pack was created for a different platform" - ); - } -} diff --git a/src/header.sh b/src/header.sh index b28ca27..cf9599b 100644 --- a/src/header.sh +++ b/src/header.sh @@ -21,11 +21,10 @@ Unpacks an environment packed with pixi-pack -f no error if environment already exists -h print this help message and exit -p ENV environment prefix, defaults to $PREFIX --i INSTALLER create the environment using the specified installer defaulting to $INSTALLER -a create an activation script to activate the environment " -while getopts ":fhai:p:" x; do +while getopts ":fha:p:" x; do case "$x" in f) FORCE=1 @@ -33,9 +32,6 @@ while getopts ":fhai:p:" x; do p) PREFIX="$OPTARG" ;; - i) - INSTALLER="$OPTARG" - ;; a) CREATE_ACTIVATION_SCRIPT=true ;; @@ -46,11 +42,6 @@ while getopts ":fhai:p:" x; do esac done -if [ "$INSTALLER" != "rattler" ] && [ "$INSTALLER" != "conda" ] && [ "$INSTALLER" != "micromamba" ]; then - echo "ERROR: Invalid installer: '$INSTALLER'" >&2 - exit 1 -fi - if [ "$FORCE" = "0" ] && [ -e "$PREFIX" ]; then echo "ERROR: File or directory already exists: '$PREFIX'" >&2 echo "If you want to update an existing environment, use the -f option." >&2 @@ -78,41 +69,23 @@ tail -n +$archive_begin "$0" | head -n $(($archive_end - $archive_begin + 1)) | echo "Creating environment using $INSTALLER" -if [ "$INSTALLER" = "rattler" ]; then - ( - ls $TEMPDIR - - export PIXI_PACK_CHANNEL_DIRECTORY=$PIXI_PACK_CHANNEL_DIRECTORY - export PIXI_PACK_METADATA_PATH=$PIXI_PACK_METADATA_PATH - export PIXI_PACK_DEFAULT_VERSION=$PIXI_PACK_DEFAULT_VERSION - - rattler_start=$(($archive_end + 2)) - - tail -n +$rattler_start "$0" > "$TEMPDIR/rattler" - chmod +x "$TEMPDIR/rattler" - - "$TEMPDIR/rattler" "unpack" "$TEMPDIR" "$PREFIX" - echo "Environment created at $PREFIX" - - if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then - "$TEMPDIR/rattler" "create-script" "$PARENT_DIR" "$PREFIX" - echo "Activation script created at $PARENT_DIR/activate.sh" - fi - ) -elif [ "$INSTALLER" = "conda" ]; then - cd $TEMPDIR - conda env create -p $PREFIX --file environment.yml - echo "Environment created at $PREFIX" -elif [ "$INSTALLER" = "micromamba" ]; then - cd $TEMPDIR - micromamba create -p $PREFIX --file environment.yml - - echo "Environment created at $PREFIX" - - if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then - micromamba shell activate -p $PREFIX > $PARENTDIR/activate.sh - echo "Activation script created at $PARENTDIR/activate.sh" - fi +ls $TEMPDIR + +export PIXI_PACK_CHANNEL_DIRECTORY=$PIXI_PACK_CHANNEL_DIRECTORY +export PIXI_PACK_METADATA_PATH=$PIXI_PACK_METADATA_PATH +export PIXI_PACK_DEFAULT_VERSION=$PIXI_PACK_DEFAULT_VERSION + +pixi_pack_start=$(($archive_end + 2)) + +tail -n +$pixi_pack_start "$0" > "$TEMPDIR/pixi-pack" +chmod +x "$TEMPDIR/pixi-pack" + +"$TEMPDIR/pixi-pack" "unpack" "$TEMPDIR" "$PREFIX" +echo "Environment created at $PREFIX" + +if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then + "$TEMPDIR/pixi-pack" "create-script" "$PARENT_DIR" "$PREFIX" + echo "Activation script created at $PARENT_DIR/activate.sh" fi cd $PARENT_DIR diff --git a/src/pack.rs b/src/pack.rs index ae17658..e93ad70 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -177,6 +177,7 @@ pub async fn pack(options: PackOptions) -> Result<()> { output_folder.path(), &options.output_file, options.create_executable, + options.platform, ) .await .map_err(|e| anyhow!("could not archive directory: {}", e))?; @@ -253,10 +254,11 @@ async fn archive_directory( input_dir: &Path, archive_target: &Path, create_executable: bool, + platform: Platform, ) -> Result<()> { if create_executable { eprintln!("📦 Creating self-extracting executable"); - create_self_extracting_executable(input_dir, archive_target).await + create_self_extracting_executable(input_dir, archive_target, platform).await } else { create_tarball(input_dir, archive_target).await } @@ -292,7 +294,11 @@ async fn create_tarball(input_dir: &Path, archive_target: &Path) -> Result<()> { Ok(()) } -async fn create_self_extracting_executable(input_dir: &Path, target: &Path) -> Result<()> { +async fn create_self_extracting_executable( + input_dir: &Path, + target: &Path, + platform: Platform, +) -> Result<()> { let tarbytes = Vec::new(); let mut archive = Builder::new(tarbytes); @@ -331,11 +337,52 @@ async fn create_self_extracting_executable(input_dir: &Path, target: &Path) -> R let executable_path = target.with_extension("sh"); - // Add the binary of extractor to the final executable - #[cfg(windows)] - const EXTRACTOR: &[u8] = include_bytes!("../extractor/target/release/extractor.exe"); - #[cfg(not(windows))] - const EXTRACTOR: &[u8] = include_bytes!("../extractor/target/release/extractor"); + // Determine the target OS and architecture + let (os, arch) = match platform { + Platform::Linux64 => ("unknown-linux-musl", "x86_64"), + Platform::LinuxAarch64 => ("unknown-linux-musl", "aarch64"), + Platform::Osx64 => ("apple-darwin", "x86_64"), + Platform::OsxArm64 => ("apple-darwin", "aarch64"), + Platform::Win64 => ("pc-windows-msvc", "x86_64"), + Platform::WinArm64 => ("pc-windows-msvc", "aarch64"), + _ => return Err(anyhow!("Unsupported platform: {}", platform)), + }; + + let executable_name = format!("pixi-pack-{}-{}", arch, os); + let extension = if os.contains("windows") { ".exe" } else { "" }; + + let version = env!("CARGO_PKG_VERSION"); + let url = format!( + "https://github.com/Quantco/pixi-pack/releases/download/v{}/{}{}", + version, executable_name, extension + ); + + eprintln!("📥 Downloading pixi-pack executable..."); + let client = reqwest::Client::new(); + let response = client.get(&url).send().await?; + if !response.status().is_success() { + return Err(anyhow!("Failed to download pixi-pack executable")); + } + + let total_size = response + .content_length() + .ok_or_else(|| anyhow!("Failed to get content length"))?; + + let bar = ProgressReporter::new(total_size); + bar.pb.set_message("Downloading"); + + let mut executable_bytes = Vec::new(); + let mut stream = response.bytes_stream(); + + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + executable_bytes.extend_from_slice(&chunk); + bar.pb.inc(chunk.len() as u64); + } + + bar.pb.finish_with_message("Download complete"); + + eprintln!("✅ Pixi-pack executable downloaded successfully"); let mut final_executable = File::create(&executable_path) .await @@ -346,16 +393,7 @@ async fn create_self_extracting_executable(input_dir: &Path, target: &Path) -> R final_executable.write_all(&compressor).await?; final_executable.write_all(b"\n").await?; final_executable.write_all(b"@@END_ARCHIVE@@\n").await?; - final_executable.write_all(EXTRACTOR).await?; - - // Make the file executable - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut perms = fs::metadata(&executable_path).await?.permissions(); - perms.set_mode(0o755); - fs::set_permissions(&executable_path, perms).await?; - } + final_executable.write_all(&executable_bytes).await?; Ok(()) } diff --git a/src/unpack.rs b/src/unpack.rs index 55fb479..8f50410 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -249,21 +249,22 @@ async fn create_prefix(channel_dir: &Path, target_prefix: &Path, cache_dir: &Pat // We can only do so by calling `get_or_fetch` on each package, which will // use the provided closure to fetch the package and insert it into the cache. package_cache - .get_or_fetch( - cache_key, - |destination| async move { - extract(&package_path, &destination).map(|_| ()) - }, - None, + .get_or_fetch( + cache_key, + move |destination| { + let package_path_clone = package_path.clone(); + async move { extract(&package_path_clone, &destination).map(|_| ()) } + }, + None, + ) + .await + .map_err(|e| { + anyhow!( + "could not extract \"{}\": {}", + repodata_record.as_ref().name.as_source(), + e ) - .await - .map_err(|e| { - anyhow!( - "could not extract \"{}\": {}", - repodata_record.as_ref().name.as_source(), - e - ) - })?; + })?; reporter.pb.inc(1); Ok::(repodata_record) From 1bf7523065c5537b1142356b4bc5e621a23a6466 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Sat, 21 Sep 2024 20:53:05 -0400 Subject: [PATCH 23/32] Added windows support --- .pre-commit-config.yaml | 2 +- Cargo.lock | 1 + Cargo.toml | 1 + src/header.ps1 | 95 +++++++++++++++++++++++++++ src/header.sh | 141 +++++++++++++++++++++++----------------- src/main.rs | 30 +++------ src/pack.rs | 49 +++++++------- 7 files changed, 215 insertions(+), 104 deletions(-) create mode 100644 src/header.ps1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 87c28f0..afd84f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,7 +17,7 @@ repos: types: [text] - id: end-of-file-fixer name: end-of-file-fixer - exclude: src/header.sh + exclude: src/header.(sh|ps1) entry: pixi run -e lint end-of-file-fixer language: system types: [text] diff --git a/Cargo.lock b/Cargo.lock index 0464c95..32b4e2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2415,6 +2415,7 @@ version = "0.1.8" dependencies = [ "anyhow", "async-std", + "base64 0.22.1", "clap", "clap-verbosity-flag", "futures", diff --git a/Cargo.toml b/Cargo.toml index f974dcb..4b6e4de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ tracing-log = "0.2.0" url = "2.5.2" fxhash = "0.2.1" tempfile = "3.12.0" +base64 = "0.22.1" [dev-dependencies] async-std = "1.12.0" diff --git a/src/header.ps1 b/src/header.ps1 new file mode 100644 index 0000000..ee29043 --- /dev/null +++ b/src/header.ps1 @@ -0,0 +1,95 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$TEMPDIR = New-TemporaryDirectory +$PREFIX = "" +$FORCE = $false +$VERBOSE = $false +$QUIET = $false +$UNPACK_SHELL = "" + +$USAGE = @" +usage: $($MyInvocation.MyCommand.Name) [options] + +Unpacks an environment packed using pixi-pack + +-f, --force No error if environment already exists +-h, --help Print this help message and exit +-o, --output-directory Where to unpack the environment +-s, --shell Sets the shell [options: bash, zsh, xonsh, cmd, powershell, fish, nushell] +-v, --verbose Increase logging verbosity +-q, --quiet Decrease logging verbosity +"@ + +$args = $MyInvocation.UnboundArguments +for ($i = 0; $i -lt $args.Count; $i++) { + switch ($args[$i]) { + "-f" { $FORCE = $true } + "--force" { $FORCE = $true } + "-o" { $PREFIX = $args[++$i] } + "--output-directory" { $PREFIX = $args[++$i] } + "-s" { $UNPACK_SHELL = $args[++$i] } + "--shell" { $UNPACK_SHELL = $args[++$i] } + "-v" { $VERBOSE = $true } + "--verbose" { $VERBOSE = $true } + "-q" { $QUIET = $true } + "--quiet" { $QUIET = $true } + "-h" { Write-Output $USAGE; exit 2 } + "--help" { Write-Output $USAGE; exit 2 } + } +} + +if (-not $FORCE -and (Test-Path $PREFIX)) { + Write-Error "ERROR: File or directory already exists: '$PREFIX'" + Write-Error "If you want to update an existing environment, use the -f option." + exit 1 +} + +if ($FORCE -and (Test-Path $PREFIX)) { + Remove-Item -Recurse -Force $PREFIX +} + +Write-Host "Unpacking payload ..." +$scriptContent = Get-Content -Raw -Path $MyInvocation.MyCommand.Path +$headerEnd = $scriptContent.IndexOf("__END_HEADER__") +$archiveEnd = $scriptContent.IndexOf("__END_ARCHIVE__", $headerEnd) + +# Extract the base64-encoded archive data between __END_HEADER__ and __END_ARCHIVE__ +$archiveContent = $scriptContent.Substring($headerEnd + "__END_HEADER__".Length, $archiveEnd - $headerEnd - "__END_HEADER__".Length) +[System.IO.File]::WriteAllBytes("$TEMPDIR\archive.tar", [System.Convert]::FromBase64String($archiveContent.Trim())) + +Write-Host "Creating environment..." + +# Extract the base64-encoded pixi-pack binary after __END_ARCHIVE__ +$pixiPackContent = $scriptContent.Substring($archiveEnd + "__END_ARCHIVE__".Length) +[System.IO.File]::WriteAllBytes("$TEMPDIR\pixi-pack.exe", [System.Convert]::FromBase64String($pixiPackContent.Trim())) + +if ($VERBOSE -and $QUIET) { + Write-Error "ERROR: Verbose and quiet options cannot be used together." + exit 1 +} + +$VERBOSITY_FLAG = "" +if ($VERBOSE) { $VERBOSITY_FLAG = "--verbose" } +if ($QUIET) { $VERBOSITY_FLAG = "--quiet" } + +$OUTPUT_DIR_FLAG = "" +if ($PREFIX) { $OUTPUT_DIR_FLAG = "--output-directory $PREFIX" } + +$SHELL_FLAG = "" +if ($UNPACK_SHELL) { $SHELL_FLAG = "--shell $UNPACK_SHELL" } + +$CMD = "& `"$TEMPDIR\pixi-pack.exe`" unpack $OUTPUT_DIR_FLAG $VERBOSITY_FLAG $SHELL_FLAG `"$TEMPDIR\archive.tar`"" + +# Execute the command +Invoke-Expression $CMD + +exit 0 + +function New-TemporaryDirectory { + $parent = [System.IO.Path]::GetTempPath() + [string] $name = [System.Guid]::NewGuid() + New-Item -ItemType Directory -Path (Join-Path $parent $name) +} + +__END_HEADER__ \ No newline at end of file diff --git a/src/header.sh b/src/header.sh index cf9599b..d107c2c 100644 --- a/src/header.sh +++ b/src/header.sh @@ -1,94 +1,117 @@ #!/bin/sh set -eu - -TEMPDIR=`mktemp -d` -PREFIX="env" +TEMPDIR=$(mktemp -d) +PREFIX="" FORCE=0 -INSTALLER="rattler" # Default to rattler -CREATE_ACTIVATION_SCRIPT=false - -# Pixi Constants ./lib.rs -PIXI_PACK_CHANNEL_DIRECTORY="" -PIXI_PACK_METADATA_PATH="" -PIXI_PACK_DEFAULT_VERSION="" +VERBOSE=0 +QUIET=0 +UNPACK_SHELL="" USAGE=" usage: $0 [options] -Unpacks an environment packed with pixi-pack +Unpacks an environment packed using pixi-pack --f no error if environment already exists --h print this help message and exit --p ENV environment prefix, defaults to $PREFIX --a create an activation script to activate the environment +-f, --force No error if environment already exists +-h, --help Print this help message and exit +-o, --output-directory Where to unpack the environment +-s, --shell Sets the shell [options: bash, zsh, xonsh, cmd, powershell, fish, nushell] +-v, --verbose Increase logging verbosity +-q, --quiet Decrease logging verbosity " - -while getopts ":fha:p:" x; do - case "$x" in - f) - FORCE=1 - ;; - p) - PREFIX="$OPTARG" - ;; - a) - CREATE_ACTIVATION_SCRIPT=true - ;; - h) - echo "$USAGE" - exit 2 - ;; - esac +# Parse command-line options +while getopts ":hfvo:s:q" opt; do + case ${opt} in + h ) + echo "$USAGE" + exit 0 + ;; + f ) + FORCE=1 + ;; + v ) + VERBOSE=1 + ;; + o ) + PREFIX="$OPTARG" + ;; + s ) + UNPACK_SHELL="$OPTARG" + ;; + q ) + QUIET=1 + ;; + \? ) + echo "Invalid option: -$OPTARG" >&2 + echo "$USAGE" >&2 + exit 1 + ;; + : ) + echo "Option -$OPTARG requires an argument" >&2 + echo "$USAGE" >&2 + exit 1 + ;; + esac done +shift $((OPTIND -1)) + +# Validate shell option if provided +if [ -n "$UNPACK_SHELL" ]; then + case "$UNPACK_SHELL" in + bash|zsh|xonsh|cmd|powershell|fish|nushell) + ;; + *) + echo "Invalid shell option: $UNPACK_SHELL" >&2 + echo "Valid options are: bash, zsh, xonsh, cmd, powershell, fish, nushell" >&2 + exit 1 + ;; + esac +fi -if [ "$FORCE" = "0" ] && [ -e "$PREFIX" ]; then +if [ "$FORCE" = "0" ] && [ -n "$PREFIX" ] && [ -e "$PREFIX" ]; then echo "ERROR: File or directory already exists: '$PREFIX'" >&2 echo "If you want to update an existing environment, use the -f option." >&2 exit 1 fi -if [ "$FORCE" = "1" ] && [ -e "$PREFIX" ]; then +if [ "$FORCE" = "1" ] && [ -n "$PREFIX" ] && [ -e "$PREFIX" ]; then rm -rf "$PREFIX" fi -if [ "$CREATE_ACTIVATION_SCRIPT" = true ] && [ "$INSTALLER" = "conda" ]; then - echo "ERROR: Activation script creation is only supported with rattler or micromamba as the installer." >&2 - exit 1 -fi - -mkdir -p "$PREFIX" -PREFIX="$(realpath "$PREFIX")" -PARENT_DIR="$(dirname "$PREFIX")" - archive_begin=$(($(grep -anm 1 "^@@END_HEADER@@" "$0" | sed 's/:.*//') + 1)) archive_end=$(($(grep -anm 1 "^@@END_ARCHIVE@@" "$0" | sed 's/:.*//') - 1)) echo "Unpacking payload ..." -tail -n +$archive_begin "$0" | head -n $(($archive_end - $archive_begin + 1)) | tar -xvf - -C "$TEMPDIR" - -echo "Creating environment using $INSTALLER" - -ls $TEMPDIR - -export PIXI_PACK_CHANNEL_DIRECTORY=$PIXI_PACK_CHANNEL_DIRECTORY -export PIXI_PACK_METADATA_PATH=$PIXI_PACK_METADATA_PATH -export PIXI_PACK_DEFAULT_VERSION=$PIXI_PACK_DEFAULT_VERSION +tail -n +$archive_begin "$0" | head -n $(($archive_end - $archive_begin + 1)) | base64 -d > "$TEMPDIR/archive.tar" pixi_pack_start=$(($archive_end + 2)) -tail -n +$pixi_pack_start "$0" > "$TEMPDIR/pixi-pack" +tail -n +$pixi_pack_start "$0" | base64 -d > "$TEMPDIR/pixi-pack" chmod +x "$TEMPDIR/pixi-pack" +if [ "$VERBOSE" = "1" ] && [ "$QUIET" = "1" ]; then + printf "ERROR: Verbose and quiet options cannot be used together.\n" >&2 + exit 1 +fi + +VERBOSITY_FLAG="" +[ "$VERBOSE" = "1" ] && VERBOSITY_FLAG="--verbose" +[ "$QUIET" = "1" ] && VERBOSITY_FLAG="--quiet" + +OUTPUT_DIR_FLAG="" +[ -n "$PREFIX" ] && OUTPUT_DIR_FLAG="--output-directory $PREFIX" -"$TEMPDIR/pixi-pack" "unpack" "$TEMPDIR" "$PREFIX" -echo "Environment created at $PREFIX" +SHELL_FLAG="" +[ -n "$UNPACK_SHELL" ] && SHELL_FLAG="--shell $UNPACK_SHELL" -if [ "$CREATE_ACTIVATION_SCRIPT" = true ]; then - "$TEMPDIR/pixi-pack" "create-script" "$PARENT_DIR" "$PREFIX" - echo "Activation script created at $PARENT_DIR/activate.sh" +CMD="\"$TEMPDIR/pixi-pack\" unpack $OUTPUT_DIR_FLAG $VERBOSITY_FLAG" +if [ -n "$UNPACK_SHELL" ]; then + CMD="$CMD --shell $UNPACK_SHELL" fi +CMD="$CMD \"$TEMPDIR/archive.tar\"" -cd $PARENT_DIR +# Execute the command +eval "$CMD" exit 0 @@END_HEADER@@ \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index 6d6a37b..a7f3b8d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -110,19 +110,15 @@ async fn main() -> Result<()> { ignore_pypi_errors, create_executable, } => { - if create_executable && is_unsupported_platform(&platform) { - return Err(anyhow::anyhow!("Creating self-extracting executables is only supported on macOS and Linux. Current platform: {}", platform)); - } - - let mut output_file_with_extension = output_file; - - if create_executable { - // TODO: Add support for other platforms - // Change this to shell.extension() - output_file_with_extension = output_file_with_extension.with_extension("sh"); + let output_file_with_extension = if create_executable { + output_file.with_extension(if platform.is_windows() { + "ps1" + } else { + "sh" + }) } else { - output_file_with_extension = output_file_with_extension.with_extension("tar"); - } + output_file.with_extension("tar") + }; let options = PackOptions { environment, @@ -158,12 +154,4 @@ async fn main() -> Result<()> { tracing::debug!("Finished running pixi-pack"); Ok(()) -} - -/// Check if the given platform supports creating self-extracting executables -fn is_unsupported_platform(platform: &Platform) -> bool { - matches!( - platform, - Platform::Win32 | Platform::Win64 | Platform::WinArm64 - ) -} +} \ No newline at end of file diff --git a/src/pack.rs b/src/pack.rs index e93ad70..9ff2635 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -13,6 +13,7 @@ use tokio::{ }; use anyhow::Result; +use base64::engine::{general_purpose::STANDARD, Engine}; use futures::{stream, StreamExt, TryFutureExt, TryStreamExt}; use rattler_conda_types::{package::ArchiveType, ChannelInfo, PackageRecord, Platform, RepoData}; use rattler_lock::{CondaPackage, LockFile, Package}; @@ -22,7 +23,7 @@ use tokio_tar::Builder; use crate::{ get_size, PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, - DEFAULT_PIXI_PACK_VERSION, PIXI_PACK_METADATA_PATH, + PIXI_PACK_METADATA_PATH, }; use anyhow::anyhow; @@ -317,25 +318,16 @@ async fn create_self_extracting_executable( .await .map_err(|e| anyhow!("could not flush output: {}", e))?; - let header = include_str!("header.sh") - .to_string() - .replace( - "PIXI_PACK_CHANNEL_DIRECTORY=\"\"", - &format!("PIXI_PACK_CHANNEL_DIRECTORY=\"{}\"", CHANNEL_DIRECTORY_NAME), - ) - .replace( - "PIXI_PACK_METADATA_PATH=\"\"", - &format!("PIXI_PACK_METADATA_PATH=\"{}\"", PIXI_PACK_METADATA_PATH), - ) - .replace( - "PIXI_PACK_DEFAULT_VERSION=\"\"", - &format!( - "PIXI_PACK_DEFAULT_VERSION=\"{}\"", - DEFAULT_PIXI_PACK_VERSION - ), - ); + let windows_header = include_str!("header.ps1"); + let unix_header = include_str!("header.sh"); + + let header = if platform.is_windows() { + windows_header + } else { + unix_header + }; - let executable_path = target.with_extension("sh"); + let executable_path = target.with_extension(if platform.is_windows() { "ps1" } else { "sh" }); // Determine the target OS and architecture let (os, arch) = match platform { @@ -349,7 +341,7 @@ async fn create_self_extracting_executable( }; let executable_name = format!("pixi-pack-{}-{}", arch, os); - let extension = if os.contains("windows") { ".exe" } else { "" }; + let extension = if platform.is_windows() { ".exe" } else { "" }; let version = env!("CARGO_PKG_VERSION"); let url = format!( @@ -390,10 +382,21 @@ async fn create_self_extracting_executable( final_executable.write_all(header.as_bytes()).await?; final_executable.write_all(b"\n").await?; // Add a newline after the header - final_executable.write_all(&compressor).await?; + + // Encode the archive to base64 + let archive_base64 = STANDARD.encode(&compressor); + final_executable.write_all(archive_base64.as_bytes()).await?; + final_executable.write_all(b"\n").await?; - final_executable.write_all(b"@@END_ARCHIVE@@\n").await?; - final_executable.write_all(&executable_bytes).await?; + if platform.is_windows() { + final_executable.write_all(b"__END_ARCHIVE__\n").await?; + } else { + final_executable.write_all(b"@@END_ARCHIVE@@\n").await?; + } + + // Encode the executable to base64 + let executable_base64 = STANDARD.encode(&executable_bytes); + final_executable.write_all(executable_base64.as_bytes()).await?; Ok(()) } From 4157a22632211a1b1174e3a899a5a8461082a14c Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Sun, 22 Sep 2024 00:29:40 -0400 Subject: [PATCH 24/32] added tests --- src/header.ps1 | 111 +++++++++++++++++++++++++------------- src/main.rs | 8 +-- src/pack.rs | 17 +++--- tests/integration_test.rs | 82 +++++++++++++++++++++------- 4 files changed, 149 insertions(+), 69 deletions(-) diff --git a/src/header.ps1 b/src/header.ps1 index ee29043..196b353 100644 --- a/src/header.ps1 +++ b/src/header.ps1 @@ -1,5 +1,9 @@ -Set-StrictMode -Version Latest -$ErrorActionPreference = "Stop" +function New-TemporaryDirectory { + $parent = [System.IO.Path]::GetTempPath() + [string] $name = [System.Guid]::NewGuid() + $tempDir = New-Item -ItemType Directory -Path (Join-Path $parent $name) + return $tempDir.FullName +} $TEMPDIR = New-TemporaryDirectory $PREFIX = "" @@ -21,6 +25,7 @@ Unpacks an environment packed using pixi-pack -q, --quiet Decrease logging verbosity "@ +# Parse command-line arguments $args = $MyInvocation.UnboundArguments for ($i = 0; $i -lt $args.Count; $i++) { switch ($args[$i]) { @@ -39,57 +44,89 @@ for ($i = 0; $i -lt $args.Count; $i++) { } } -if (-not $FORCE -and (Test-Path $PREFIX)) { - Write-Error "ERROR: File or directory already exists: '$PREFIX'" - Write-Error "If you want to update an existing environment, use the -f option." +# Check if verbose and quiet are both set +if ($VERBOSE -and $QUIET) { + Write-Error "ERROR: Verbose and quiet options cannot be used together." exit 1 } -if ($FORCE -and (Test-Path $PREFIX)) { - Remove-Item -Recurse -Force $PREFIX -} - -Write-Host "Unpacking payload ..." +# Step 1: Extract the archive and pixi-pack executable, and decode them $scriptContent = Get-Content -Raw -Path $MyInvocation.MyCommand.Path -$headerEnd = $scriptContent.IndexOf("__END_HEADER__") -$archiveEnd = $scriptContent.IndexOf("__END_ARCHIVE__", $headerEnd) +$lines = $scriptContent -split "`r?`n" -# Extract the base64-encoded archive data between __END_HEADER__ and __END_ARCHIVE__ -$archiveContent = $scriptContent.Substring($headerEnd + "__END_HEADER__".Length, $archiveEnd - $headerEnd - "__END_HEADER__".Length) -[System.IO.File]::WriteAllBytes("$TEMPDIR\archive.tar", [System.Convert]::FromBase64String($archiveContent.Trim())) +$headerLine = $null +$archiveLine = $null -Write-Host "Creating environment..." +# Find the lines where __END_HEADER__ and __END_ARCHIVE__ occur +for ($i = 0; $i -lt $lines.Count; $i++) { + if ($lines[$i] -like "*__END_HEADER__*") { + $headerLine = $i + 1 + } + if ($lines[$i] -like "*__END_ARCHIVE__*") { + $archiveLine = $i + 1 + } +} -# Extract the base64-encoded pixi-pack binary after __END_ARCHIVE__ -$pixiPackContent = $scriptContent.Substring($archiveEnd + "__END_ARCHIVE__".Length) -[System.IO.File]::WriteAllBytes("$TEMPDIR\pixi-pack.exe", [System.Convert]::FromBase64String($pixiPackContent.Trim())) +if (-not $headerLine -or -not $archiveLine) { + Write-Error "Markers __END_HEADER__ or __END_ARCHIVE__ not found." + exit 1 +} -if ($VERBOSE -and $QUIET) { - Write-Error "ERROR: Verbose and quiet options cannot be used together." +# Extract Base64 content for the tar archive +$archiveContent = $lines[($headerLine)..($archiveLine - 2)] -join "" +$archiveContent = $archiveContent.Trim() + +# Decode Base64 content into tar file +try { + $decodedArchive = [System.Convert]::FromBase64String($archiveContent) + $archivePath = "$TEMPDIR\archive.tar" + [System.IO.File]::WriteAllBytes($archivePath, $decodedArchive) +} catch { + Write-Error "Failed to decode Base64 archive content: $_" exit 1 } -$VERBOSITY_FLAG = "" -if ($VERBOSE) { $VERBOSITY_FLAG = "--verbose" } -if ($QUIET) { $VERBOSITY_FLAG = "--quiet" } +# Extract Base64 content for pixi-pack executable +$pixiPackContent = $lines[($archiveLine)..($lines.Count - 1)] -join "" +$pixiPackContent = $pixiPackContent.Trim() + +# Decode Base64 content into the pixi-pack executable file +try { + $decodedPixiPack = [System.Convert]::FromBase64String($pixiPackContent) + $pixiPackPath = "$TEMPDIR\pixi-pack.exe" + [System.IO.File]::WriteAllBytes($pixiPackPath, $decodedPixiPack) +} catch { + Write-Error "Failed to decode Base64 pixi-pack content: $_" + exit 1 +} -$OUTPUT_DIR_FLAG = "" -if ($PREFIX) { $OUTPUT_DIR_FLAG = "--output-directory $PREFIX" } +# Step 2: Build the command with flags +$arguments = @("unpack") -$SHELL_FLAG = "" -if ($UNPACK_SHELL) { $SHELL_FLAG = "--shell $UNPACK_SHELL" } +# Use $PREFIX for output directory if it is provided +if ($PREFIX) { + $arguments += "--output-directory" + $arguments += $PREFIX +} -$CMD = "& `"$TEMPDIR\pixi-pack.exe`" unpack $OUTPUT_DIR_FLAG $VERBOSITY_FLAG $SHELL_FLAG `"$TEMPDIR\archive.tar`"" +# Handle verbosity/quiet flags +if ($VERBOSE) { + $arguments += "--verbose" +} elseif ($QUIET) { + $arguments += "--quiet" +} -# Execute the command -Invoke-Expression $CMD +# Add shell flag if provided +if ($UNPACK_SHELL) { + $arguments += "--shell" + $arguments += $UNPACK_SHELL +} -exit 0 +# Finally, add the path to the archive +$arguments += $archivePath -function New-TemporaryDirectory { - $parent = [System.IO.Path]::GetTempPath() - [string] $name = [System.Guid]::NewGuid() - New-Item -ItemType Directory -Path (Join-Path $parent $name) -} +& $pixiPackPath @arguments + +exit 0 __END_HEADER__ \ No newline at end of file diff --git a/src/main.rs b/src/main.rs index a7f3b8d..f69c7c9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -111,11 +111,7 @@ async fn main() -> Result<()> { create_executable, } => { let output_file_with_extension = if create_executable { - output_file.with_extension(if platform.is_windows() { - "ps1" - } else { - "sh" - }) + output_file.with_extension(if platform.is_windows() { "ps1" } else { "sh" }) } else { output_file.with_extension("tar") }; @@ -154,4 +150,4 @@ async fn main() -> Result<()> { tracing::debug!("Finished running pixi-pack"); Ok(()) -} \ No newline at end of file +} diff --git a/src/pack.rs b/src/pack.rs index 9ff2635..badbfe5 100644 --- a/src/pack.rs +++ b/src/pack.rs @@ -22,8 +22,7 @@ use reqwest_middleware::ClientWithMiddleware; use tokio_tar::Builder; use crate::{ - get_size, PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, - PIXI_PACK_METADATA_PATH, + get_size, PixiPackMetadata, ProgressReporter, CHANNEL_DIRECTORY_NAME, PIXI_PACK_METADATA_PATH, }; use anyhow::anyhow; @@ -382,21 +381,25 @@ async fn create_self_extracting_executable( final_executable.write_all(header.as_bytes()).await?; final_executable.write_all(b"\n").await?; // Add a newline after the header - + // Encode the archive to base64 let archive_base64 = STANDARD.encode(&compressor); - final_executable.write_all(archive_base64.as_bytes()).await?; - + final_executable + .write_all(archive_base64.as_bytes()) + .await?; + final_executable.write_all(b"\n").await?; if platform.is_windows() { final_executable.write_all(b"__END_ARCHIVE__\n").await?; } else { final_executable.write_all(b"@@END_ARCHIVE@@\n").await?; } - + // Encode the executable to base64 let executable_base64 = STANDARD.encode(&executable_bytes); - final_executable.write_all(executable_base64.as_bytes()).await?; + final_executable + .write_all(executable_base64.as_bytes()) + .await?; Ok(()) } diff --git a/tests/integration_test.rs b/tests/integration_test.rs index f8f96b3..c1684aa 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -5,7 +5,7 @@ use std::{path::PathBuf, process::Command}; use pixi_pack::{unarchive, PackOptions, PixiPackMetadata, UnpackOptions}; use rattler_conda_types::Platform; use rattler_conda_types::RepoData; -use rattler_shell::shell::{Bash, ShellEnum}; +use rattler_shell::shell::{Bash, PowerShell, ShellEnum}; use rstest::*; use tempfile::{tempdir, TempDir}; use tokio::fs::File; @@ -274,13 +274,16 @@ async fn test_pypi_ignore( let pack_result = pixi_pack::pack(pack_options).await; assert_eq!(pack_result.is_err(), should_fail); } - #[rstest] #[tokio::test] async fn test_create_executable(options: Options, required_fs_objects: Vec<&'static str>) { let mut pack_options = options.pack_options; pack_options.create_executable = true; - pack_options.output_file = options.output_dir.path().join("environment.sh"); + pack_options.output_file = options.output_dir.path().join(if cfg!(windows) { + "environment.ps1" + } else { + "environment.sh" + }); let pack_file = pack_options.output_file.clone(); @@ -288,7 +291,10 @@ async fn test_create_executable(options: Options, required_fs_objects: Vec<&'sta assert!(pack_result.is_ok(), "{:?}", pack_result); assert!(pack_file.exists()); - assert_eq!(pack_file.extension().unwrap(), "sh"); + assert_eq!( + pack_file.extension().unwrap(), + if cfg!(windows) { "ps1" } else { "sh" } + ); #[cfg(unix)] { @@ -303,7 +309,11 @@ async fn test_create_executable(options: Options, required_fs_objects: Vec<&'sta let unpack_options = UnpackOptions { pack_file, output_directory: unpack_dir_path.to_path_buf(), - shell: Some(ShellEnum::Bash(Bash)), + shell: Some(if cfg!(windows) { + ShellEnum::PowerShell(PowerShell::default()) + } else { + ShellEnum::Bash(Bash) + }), }; let unpack_result = pixi_pack::unpack(unpack_options).await; @@ -312,7 +322,11 @@ async fn test_create_executable(options: Options, required_fs_objects: Vec<&'sta let env_dir = unpack_dir_path.join("env"); assert!(env_dir.exists()); - let activate_file = unpack_dir_path.join("activate.sh"); + let activate_file = unpack_dir_path.join(if cfg!(windows) { + "activate.ps1" + } else { + "activate.sh" + }); assert!(activate_file.exists()); required_fs_objects @@ -325,11 +339,19 @@ async fn test_create_executable(options: Options, required_fs_objects: Vec<&'sta #[rstest] #[tokio::test] -#[cfg(any(target_os = "linux", target_os = "macos"))] +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<&'static str>) { let mut pack_options = options.pack_options; pack_options.create_executable = true; - pack_options.output_file = options.output_dir.path().join("environment.sh"); + + #[cfg(target_os = "windows")] + { + pack_options.output_file = options.output_dir.path().join("environment.ps1"); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + pack_options.output_file = options.output_dir.path().join("environment.sh"); + } let pack_file = pack_options.output_file.clone(); @@ -337,24 +359,46 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& assert!(pack_result.is_ok(), "{:?}", pack_result); assert!(pack_file.exists()); - assert_eq!(pack_file.extension().unwrap(), "sh"); - let output = Command::new("sh") - .arg(pack_file) - .arg("-af") - .arg("-p") - .arg(options.output_dir.path().join("env")) - .output() - .expect("Failed to execute packed file for extraction"); - - assert!(output.status.success(), "Extraction failed: {:?}", output); + #[cfg(target_os = "windows")] + { + assert_eq!(pack_file.extension().unwrap(), "ps1"); + let output = Command::new("powershell") + .arg("-ExecutionPolicy") + .arg("Bypass") + .arg("-File") + .arg(&pack_file) + .arg("-f") + .arg("-o") + .arg(options.output_dir.path().join("env")) + .output() + .expect("Failed to execute packed file for extraction"); + assert!(output.status.success(), "Extraction failed: {:?}", output); + } + #[cfg(any(target_os = "linux", target_os = "macos"))] + { + assert_eq!(pack_file.extension().unwrap(), "sh"); + let output = Command::new("sh") + .arg(pack_file) + .arg("-f") + .arg("-o") + .arg(options.output_dir.path().join("env")) + .output() + .expect("Failed to execute packed file for extraction"); + assert!(output.status.success(), "Extraction failed: {:?}", output); + } let env_dir = options.output_dir.path().join("env"); assert!( env_dir.exists(), "Environment directory not found after extraction" ); - let activate_file = options.output_dir.path().join("activate.sh"); + + #[cfg(target_os = "windows")] + let activate_file = env_dir.join("activate.bat"); + #[cfg(any(target_os = "linux", target_os = "macos"))] + let activate_file = env_dir.join("activate.sh"); + assert!( activate_file.exists(), "Activation script not found after extraction" From 11093602260e9b20cce82f97793cf96ec001dd48 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Sun, 22 Sep 2024 11:14:10 -0400 Subject: [PATCH 25/32] added tests and run fmt --- tests/integration_test.rs | 110 +++++++++----------------------------- 1 file changed, 26 insertions(+), 84 deletions(-) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index c1684aa..75cd2e1 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -5,7 +5,7 @@ use std::{path::PathBuf, process::Command}; use pixi_pack::{unarchive, PackOptions, PixiPackMetadata, UnpackOptions}; use rattler_conda_types::Platform; use rattler_conda_types::RepoData; -use rattler_shell::shell::{Bash, PowerShell, ShellEnum}; +use rattler_shell::shell::{Bash, ShellEnum}; use rstest::*; use tempfile::{tempdir, TempDir}; use tokio::fs::File; @@ -276,81 +276,18 @@ async fn test_pypi_ignore( } #[rstest] #[tokio::test] -async fn test_create_executable(options: Options, required_fs_objects: Vec<&'static str>) { - let mut pack_options = options.pack_options; - pack_options.create_executable = true; - pack_options.output_file = options.output_dir.path().join(if cfg!(windows) { - "environment.ps1" - } else { - "environment.sh" - }); - - let pack_file = pack_options.output_file.clone(); - - let pack_result = pixi_pack::pack(pack_options).await; - assert!(pack_result.is_ok(), "{:?}", pack_result); - - assert!(pack_file.exists()); - assert_eq!( - pack_file.extension().unwrap(), - if cfg!(windows) { "ps1" } else { "sh" } - ); - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let metadata = std::fs::metadata(&pack_file).unwrap(); - let permissions = metadata.permissions(); - assert!(permissions.mode() & 0o111 != 0); - } - - let unpack_dir = tempdir().expect("Couldn't create a temp dir for tests"); - let unpack_dir_path = unpack_dir.path(); - let unpack_options = UnpackOptions { - pack_file, - output_directory: unpack_dir_path.to_path_buf(), - shell: Some(if cfg!(windows) { - ShellEnum::PowerShell(PowerShell::default()) - } else { - ShellEnum::Bash(Bash) - }), - }; - - let unpack_result = pixi_pack::unpack(unpack_options).await; - assert!(unpack_result.is_ok(), "{:?}", unpack_result); - - let env_dir = unpack_dir_path.join("env"); - assert!(env_dir.exists()); - - let activate_file = unpack_dir_path.join(if cfg!(windows) { - "activate.ps1" - } else { - "activate.sh" - }); - assert!(activate_file.exists()); - - required_fs_objects - .iter() - .map(|dir| env_dir.join(dir)) - .for_each(|dir| { - assert!(dir.exists(), "{:?} does not exist", dir); - }); -} - -#[rstest] -#[tokio::test] -#[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<&'static str>) { + let temp_dir = tempfile::tempdir().expect("Couldn't create a temp dir for tests"); let mut pack_options = options.pack_options; pack_options.create_executable = true; #[cfg(target_os = "windows")] { - pack_options.output_file = options.output_dir.path().join("environment.ps1"); + pack_options.output_file = temp_dir.path().join("environment.ps1"); } #[cfg(any(target_os = "linux", target_os = "macos"))] { - pack_options.output_file = options.output_dir.path().join("environment.sh"); + pack_options.output_file = temp_dir.path().join("environment.sh"); } let pack_file = pack_options.output_file.clone(); @@ -358,34 +295,36 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& let pack_result = pixi_pack::pack(pack_options).await; assert!(pack_result.is_ok(), "{:?}", pack_result); - assert!(pack_file.exists()); + assert!( + pack_file.exists(), + "Pack file does not exist at {:?}", + pack_file + ); - #[cfg(target_os = "windows")] + #[cfg(any(target_os = "linux", target_os = "macos"))] { - assert_eq!(pack_file.extension().unwrap(), "ps1"); - let output = Command::new("powershell") - .arg("-ExecutionPolicy") - .arg("Bypass") - .arg("-File") + assert_eq!(pack_file.extension().unwrap(), "sh"); + let output = Command::new("sh") .arg(&pack_file) .arg("-f") .arg("-o") - .arg(options.output_dir.path().join("env")) + .arg(options.output_dir.path()) .output() .expect("Failed to execute packed file for extraction"); - assert!(output.status.success(), "Extraction failed: {:?}", output); + assert!(output.status.success(), "Packed file execution failed"); } - #[cfg(any(target_os = "linux", target_os = "macos"))] + #[cfg(target_os = "windows")] { - assert_eq!(pack_file.extension().unwrap(), "sh"); - let output = Command::new("sh") - .arg(pack_file) + assert_eq!(pack_file.extension().unwrap(), "ps1"); + let output = Command::new("powershell") + .arg("-File") + .arg(&pack_file) .arg("-f") .arg("-o") - .arg(options.output_dir.path().join("env")) + .arg(options.output_dir.path()) .output() .expect("Failed to execute packed file for extraction"); - assert!(output.status.success(), "Extraction failed: {:?}", output); + assert!(output.status.success(), "Packed file execution failed"); } let env_dir = options.output_dir.path().join("env"); @@ -395,9 +334,9 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& ); #[cfg(target_os = "windows")] - let activate_file = env_dir.join("activate.bat"); + let activate_file = options.output_dir.path().join("activate.bat"); #[cfg(any(target_os = "linux", target_os = "macos"))] - let activate_file = env_dir.join("activate.sh"); + let activate_file = options.output_dir.path().join("activate.sh"); assert!( activate_file.exists(), @@ -410,4 +349,7 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& .for_each(|dir| { assert!(dir.exists(), "{:?} does not exist", dir); }); + + // Keep the temporary directory alive until the end of the test + drop(temp_dir); } From 5c06d2b9a88c309f6aee6ae9638c34e443a11302 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Sun, 22 Sep 2024 11:18:08 -0400 Subject: [PATCH 26/32] remove extra function in unpack --- src/unpack.rs | 58 +++++---------------------------------------------- 1 file changed, 5 insertions(+), 53 deletions(-) diff --git a/src/unpack.rs b/src/unpack.rs index 8f50410..8f9a174 100644 --- a/src/unpack.rs +++ b/src/unpack.rs @@ -1,7 +1,4 @@ -use std::{ - io::Cursor, - path::{Path, PathBuf}, -}; +use std::path::{Path, PathBuf}; use anyhow::{anyhow, Result}; use futures::{ @@ -21,7 +18,6 @@ use rattler_shell::{ }; use tokio::fs; -use tokio::io::AsyncReadExt; use tokio_stream::wrappers::ReadDirStream; use tokio_tar::Archive; use url::Url; @@ -46,15 +42,10 @@ pub async fn unpack(options: UnpackOptions) -> Result<()> { let unpack_dir = tmp_dir.path(); tracing::info!("Unarchiving pack to {}", unpack_dir.display()); - if options.pack_file.extension().unwrap_or_default() == "sh" { - unarchive_from_shellscript(&options.pack_file, unpack_dir) - .await - .map_err(|e| anyhow!("Could not extract archive from shell script: {}", e))?; - } else { - unarchive(&options.pack_file, unpack_dir) - .await - .map_err(|e| anyhow!("Could not unarchive: {}", e))?; - } + + unarchive(&options.pack_file, unpack_dir) + .await + .map_err(|e| anyhow!("Could not unarchive: {}", e))?; validate_metadata_file(unpack_dir.join(PIXI_PACK_METADATA_PATH)).await?; @@ -157,45 +148,6 @@ async fn collect_packages(channel_dir: &Path) -> Result Result<()> { - let mut shell_script = fs::File::open(shell_script_path) - .await - .map_err(|e| anyhow!("could not open shell script: {}", e))?; - - let mut content = Vec::new(); - shell_script - .read_to_end(&mut content) - .await - .map_err(|e| anyhow!("could not read shell script: {}", e))?; - - let end_header = b"@@END_HEADER@@\n"; - let end_archive = b"@@END_ARCHIVE@@\n"; - - let start = content - .windows(end_header.len()) - .position(|window| window == end_header) - .map(|pos| pos + end_header.len()) - .ok_or_else(|| anyhow!("Could not find @@END_HEADER@@ in shell script"))?; - - let end = content - .windows(end_archive.len()) - .position(|window| window == end_archive) - .ok_or_else(|| anyhow!("Could not find @@END_ARCHIVE@@ in shell script"))?; - - let archive_content = &content[start..end]; - - let reader = tokio::io::BufReader::new(Cursor::new(archive_content)); - let mut archive = Archive::new(reader); - - archive - .unpack(target_dir) - .await - .map_err(|e| anyhow!("could not unpack archive: {}", e))?; - - Ok(()) -} - /// Unarchive a tarball. pub async fn unarchive(archive_path: &Path, target_dir: &Path) -> Result<()> { let file = fs::File::open(archive_path) From 76d41fc6b80f00cb0db9c073858d5a86abe69fa2 Mon Sep 17 00:00:00 2001 From: prsabahrami Date: Sun, 22 Sep 2024 11:20:27 -0400 Subject: [PATCH 27/32] Remove extractor tests --- examples/webserver/my_webserver/__init__.py | 8 ++++++++ pixi.toml | 4 +--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 examples/webserver/my_webserver/__init__.py diff --git a/examples/webserver/my_webserver/__init__.py b/examples/webserver/my_webserver/__init__.py new file mode 100644 index 0000000..52f0d71 --- /dev/null +++ b/examples/webserver/my_webserver/__init__.py @@ -0,0 +1,8 @@ +from fastapi import FastAPI + +app = FastAPI() + + +@app.get("/") +def hello(): + return "Hello, World!" diff --git a/pixi.toml b/pixi.toml index 5686dd8..79b8e75 100644 --- a/pixi.toml +++ b/pixi.toml @@ -5,9 +5,7 @@ platforms = ["osx-arm64", "osx-64", "linux-64", "linux-aarch64", "win-64"] [tasks] build = "cargo build --release" -test-pixi-pack = "cargo test" -test-extractor = { cmd = "cargo test --manifest-path extractor/Cargo.toml", env = { PIXI_PACK_DEFAULT_VERSION = "1" } } -test = { depends-on = ["test-pixi-pack", "test-extractor"] } +test = "cargo test" [dependencies] rust = "1.77.2" From db13b4a947de308b0f29f72d3fb4b82784441652 Mon Sep 17 00:00:00 2001 From: Parsa Bahraminejad Date: Fri, 11 Oct 2024 18:09:44 -0400 Subject: [PATCH 28/32] Update src/header.sh Co-authored-by: Pavel Zwerschke --- src/header.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/header.sh b/src/header.sh index d107c2c..7b1d35e 100644 --- a/src/header.sh +++ b/src/header.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu -TEMPDIR=$(mktemp -d) +TEMPDIR="$(mktemp -d)" PREFIX="" FORCE=0 VERBOSE=0 From 9d2c1ac1771fa1e50ebf1adb566cc03b807708fa Mon Sep 17 00:00:00 2001 From: Parsa Bahraminejad Date: Fri, 11 Oct 2024 18:10:05 -0400 Subject: [PATCH 29/32] Update src/header.sh Co-authored-by: Pavel Zwerschke --- src/header.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/header.sh b/src/header.sh index 7b1d35e..c1bb718 100644 --- a/src/header.sh +++ b/src/header.sh @@ -1,6 +1,6 @@ #!/bin/sh -set -eu +set -euo pipefail TEMPDIR="$(mktemp -d)" PREFIX="" FORCE=0 From e980ec14fdda503c5f20a216237d33ef1e69287d Mon Sep 17 00:00:00 2001 From: Parsa Bahraminejad Date: Fri, 11 Oct 2024 18:10:28 -0400 Subject: [PATCH 30/32] Update src/main.rs Co-authored-by: Pavel Zwerschke --- src/main.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.rs b/src/main.rs index f69c7c9..37f1ffd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -63,7 +63,6 @@ enum Commands { ignore_pypi_errors: bool, /// Create self-extracting executable - /// This feature is only available on macOS and Linux. #[arg(long, default_value = "false")] create_executable: bool, }, From bc037b58bcdb96abeaddef4feb42f9a67ef8f4f8 Mon Sep 17 00:00:00 2001 From: Parsa Bahraminejad Date: Fri, 11 Oct 2024 18:11:04 -0400 Subject: [PATCH 31/32] Update tests/integration_test.rs Co-authored-by: Pavel Zwerschke --- tests/integration_test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index 75cd2e1..f340ee9 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -277,7 +277,7 @@ async fn test_pypi_ignore( #[rstest] #[tokio::test] async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<&'static str>) { - let temp_dir = tempfile::tempdir().expect("Couldn't create a temp dir for tests"); + let temp_dir = tempfile::tempdir().unwrap(); let mut pack_options = options.pack_options; pack_options.create_executable = true; From cc6b42e6842bbf0ff794d33fb30f999c47813313 Mon Sep 17 00:00:00 2001 From: Parsa Bahraminejad Date: Fri, 11 Oct 2024 18:13:18 -0400 Subject: [PATCH 32/32] Update tests/integration_test.rs Co-authored-by: Pavel Zwerschke --- tests/integration_test.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration_test.rs b/tests/integration_test.rs index f340ee9..29a9c57 100644 --- a/tests/integration_test.rs +++ b/tests/integration_test.rs @@ -334,12 +334,12 @@ async fn test_run_packed_executable(options: Options, required_fs_objects: Vec<& ); #[cfg(target_os = "windows")] - let activate_file = options.output_dir.path().join("activate.bat"); + let activation_script = options.output_dir.path().join("activate.bat"); #[cfg(any(target_os = "linux", target_os = "macos"))] - let activate_file = options.output_dir.path().join("activate.sh"); + let activation_script = options.output_dir.path().join("activate.sh"); assert!( - activate_file.exists(), + activation_script.exists(), "Activation script not found after extraction" );