diff --git a/.gitignore b/.gitignore index 5e5a1a0e1..daed69ebe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ /target/ .ignore/ -twoliter-attributions.tar.gz +*.tar.gz +/tools/krane/go-containerregistry-* +/tools/krane/krane +/tools/krane/krane.gz diff --git a/Cargo.lock b/Cargo.lock index 86472db8a..734868ed4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1815,7 +1815,10 @@ dependencies = [ name = "integration-tests" version = "0.1.0" dependencies = [ + "libc", + "tempfile", "tokio", + "toml", "twoliter", ] @@ -1892,6 +1895,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "krane-bundle" +version = "0.1.0" +dependencies = [ + "anyhow", + "flate2", + "lazy_static", + "pentacle", + "tar", +] + [[package]] name = "kube" version = "0.88.1" @@ -2150,6 +2164,7 @@ name = "oci-cli-wrapper" version = "0.1.0" dependencies = [ "async-trait", + "krane-bundle", "log", "olpc-cjson", "regex", @@ -2288,6 +2303,16 @@ dependencies = [ "serde", ] +[[package]] +name = "pentacle" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e875807b4510e6847d4ef7674ab9b3efe30cc99b933f2e6e82f6ef38f7e5352" +dependencies = [ + "libc", + "log", +] + [[package]] name = "percent-encoding" version = "2.3.1" diff --git a/Cargo.toml b/Cargo.toml index f09b1e603..949e54c2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ bottlerocket-types = { version = "0.0.14", git = "https://github.com/bottlerocke bottlerocket-variant = { version = "0.1", path = "tools/bottlerocket-variant" } buildsys = { version = "0.1", path = "tools/buildsys", lib = true, artifact = [ "bin:buildsys" ] } buildsys-config = { version = "0.1", path = "tools/buildsys-config" } +krane-bundle = { version = "0.1", path = "tools/krane" } oci-cli-wrapper = { version = "0.1", path = "tools/oci-cli-wrapper" } parse-datetime = { version = "0.1", path = "tools/parse-datetime" } pipesys = { version = "0.1", path = "tools/pipesys", lib = true, artifact = [ "bin:pipesys" ] } @@ -88,15 +89,18 @@ home = "0.5" indicatif = "0.17" inotify = "0.10.2" lazy_static = "1" +libc = "0.2" log = "0.4" maplit = "1" nix = "0.28" nonzero_ext = "0.3" num_cpus = "1" olpc-cjson = "0.1" +pentacle = "1.1" rand = { version = "0.8", default-features = false } regex = "1" reqwest = { version = "0.11", default-features = false } +seccompiler = "0.4" semver = "1" serde = "1" serde_json = "1" diff --git a/tests/integration-tests/Cargo.toml b/tests/integration-tests/Cargo.toml index 7597899f4..4343c3da6 100644 --- a/tests/integration-tests/Cargo.toml +++ b/tests/integration-tests/Cargo.toml @@ -5,5 +5,8 @@ edition = "2021" license = "MIT OR Apache-2.0" [dev-dependencies] +libc.workspace = true +tempfile.workspace = true tokio = { workspace = true, features = ["fs", "process", "rt-multi-thread"] } +toml.workspace = true twoliter = { workspace = true } diff --git a/tests/integration-tests/src/lib.rs b/tests/integration-tests/src/lib.rs index 379a7b0f2..7c4dd372c 100644 --- a/tests/integration-tests/src/lib.rs +++ b/tests/integration-tests/src/lib.rs @@ -2,7 +2,8 @@ use std::ffi::OsStr; use std::path::PathBuf; -use tokio::process::Command; +use std::process::Command; +use tempfile::TempDir; mod twoliter_update; @@ -14,7 +15,7 @@ pub fn test_projects_dir() -> PathBuf { p.join("projects") } -pub async fn run_command(cmd: S, args: I, env: E) -> std::process::Output +pub fn run_command(cmd: S, args: I, env: E) -> std::process::Output where I: IntoIterator, E: IntoIterator, @@ -35,10 +36,88 @@ where .args(args.into_iter()) .envs(env.into_iter()) .output() - .await .expect("failed to execute process"); println!("stdout: {}", String::from_utf8_lossy(&output.stdout)); println!("stderr: {}", String::from_utf8_lossy(&output.stderr)); output } + +struct KitRegistry { + temp_dir: TempDir, + container_id: String, +} + +impl KitRegistry { + fn new() -> Self { + let temp_dir = TempDir::new().expect("failed to create path for oci registry spinup"); + + let cert_dir = temp_dir.path().join("certs"); + let cert_file = cert_dir.join("registry.crt"); + std::fs::create_dir_all(&cert_dir).expect("failed to create nginx dir"); + let output = run_command( + "openssl", + [ + "req", + "-x509", + "-nodes", + "-days", + "365", + "-newkey", + "rsa:2048", + "-keyout", + cert_dir.join("registry.key").to_str().unwrap(), + "-out", + cert_file.to_str().unwrap(), + "-batch", + "-addext", + "subjectAltName=DNS:localhost", + ], + [], + ); + assert!( + output.status.success(), + "generate openssl self-signed certificates" + ); + + let output = run_command( + "docker", + [ + "run", + "-d", + "--rm", + "--volume", + "./certs:/auth/certs", + "-e REGISTRY_HTTP_RELATIVEURLS=true", + "-e REGISTRY_HTTP_ADDR=0.0.0.0:5000", + "-e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/certs/registry.crt", + "-e REGISTRY_HTTP_TLS_KEY=/auth/certs/registry.key", + "-p", + "5000:5000", + "public.ecr.aws/docker/library/registry:2.8.3", + ], + [], + ); + assert!(output.status.success(), "failed to start oci registry"); + let container_id = String::from_utf8(output.stdout).unwrap().trim().to_string(); + + Self { + temp_dir, + container_id, + } + } + + fn cert_file(&self) -> PathBuf { + self.temp_dir + .path() + .join("certs/registry.crt") + .to_path_buf() + } +} + +impl Drop for KitRegistry { + fn drop(&mut self) { + let output = run_command("docker", ["kill", &self.container_id], []); + assert!(output.status.success(), "failed to stop oci registry"); + } +} diff --git a/tests/integration-tests/src/twoliter_update.rs b/tests/integration-tests/src/twoliter_update.rs index 4672f291a..4c7f203c0 100644 --- a/tests/integration-tests/src/twoliter_update.rs +++ b/tests/integration-tests/src/twoliter_update.rs @@ -1,31 +1,32 @@ -use super::{run_command, test_projects_dir, TWOLITER_PATH}; - -const EXPECTED_LOCKFILE: &str = r#"schema-version = 1 - -[sdk] -name = "bottlerocket-sdk" -version = "0.42.0" -vendor = "bottlerocket" -source = "public.ecr.aws/bottlerocket/bottlerocket-sdk:v0.42.0" -digest = "myHHKE41h9qfeyR6V6HB0BfiLPwj3QEFLUFy4TXcR10=" - -[[kit]] -name = "bottlerocket-core-kit" -version = "2.0.0" -vendor = "custom-vendor" -source = "public.ecr.aws/bottlerocket/bottlerocket-core-kit:v2.0.0" -digest = "vlTsAAbSCzXFZofVmw8pLLkRjnG/y8mtb2QsQBSz1zk=" +use super::{run_command, test_projects_dir, KitRegistry, TWOLITER_PATH}; + +const INFRA_TOML: &str = r#" +[vendor.bottlerocket] +registry = "localhost:5000" +"#; + +const TWOLITER_OVERRIDE: &str = r#" +[custom-vendor.core-kit] +registry = "localhost:5000" +name = "core-kit-overridden" "#; -#[tokio::test] +#[test] #[ignore] -/// Generates a Twoliter.lock file for the `external-kit` project using docker -async fn test_twoliter_update_docker() { +/// Generates a Twoliter.lock file for the `external-kit` project using crane +fn test_twoliter_build_and_update() { let external_kit = test_projects_dir().join("external-kit"); - let lockfile = external_kit.join("Twoliter.lock"); - tokio::fs::remove_file(&lockfile).await.ok(); + std::fs::remove_file(&lockfile).ok(); + let override_file = external_kit.join("Twoliter.override"); + std::fs::remove_file(&override_file).ok(); + + // Build & push a local kit to the registry + let registry = KitRegistry::new(); + LocalKit::build(®istry); + // Point twoliter to the local registry as an override + std::fs::write(&override_file, TWOLITER_OVERRIDE).unwrap(); let output = run_command( TWOLITER_PATH, [ @@ -33,42 +34,101 @@ async fn test_twoliter_update_docker() { "--project-path", external_kit.join("Twoliter.toml").to_str().unwrap(), ], - [("TWOLITER_KIT_IMAGE_TOOL", "docker")], - ) - .await; + [ + ("TWOLITER_KIT_IMAGE_TOOL", "crane"), + ("SSL_CERT_FILE", registry.cert_file().to_str().unwrap()), + ], + ); assert!(output.status.success()); - let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap(); - assert_eq!(lock_contents, EXPECTED_LOCKFILE); + // Assert that we successfully create a lock + let lock_contents = std::fs::read_to_string(&lockfile).unwrap(); + let parsed: toml::Value = toml::from_str(&lock_contents).unwrap(); + let kits = parsed + .as_table() + .unwrap() + .get("kit") + .unwrap() + .as_array() + .unwrap(); + + assert_eq!(kits.len(), 1); + let core_kit = kits[0].as_table().unwrap(); + assert_eq!(core_kit.get("name").unwrap().as_str().unwrap(), "core-kit"); + assert_eq!(core_kit.get("version").unwrap().as_str().unwrap(), "1.0.0"); + assert_eq!( + core_kit.get("vendor").unwrap().as_str().unwrap(), + "custom-vendor" + ); + assert_eq!( + core_kit.get("source").unwrap().as_str().unwrap(), + "definitely-wont-resolve/core-kit:v1.0.0" + ); - tokio::fs::remove_file(&lockfile).await.ok(); + std::fs::remove_file(&lockfile).ok(); + std::fs::remove_file(&override_file).ok(); } -#[tokio::test] -#[ignore] -/// Generates a Twoliter.lock file for the `external-kit` project using crane -async fn test_twoliter_update_crane() { - let external_kit = test_projects_dir().join("external-kit"); +struct LocalKit; - let lockfile = external_kit.join("Twoliter.lock"); - tokio::fs::remove_file(&lockfile).await.ok(); +impl LocalKit { + fn build(registry: &KitRegistry) { + let local_kit = test_projects_dir().join("local-kit"); - let output = run_command( - TWOLITER_PATH, - [ - "update", - "--project-path", - external_kit.join("Twoliter.toml").to_str().unwrap(), - ], - [("TWOLITER_KIT_IMAGE_TOOL", "crane")], - ) - .await; + run_command( + TWOLITER_PATH, + [ + "update", + "--project-path", + local_kit.join("Twoliter.toml").to_str().unwrap(), + ], + [], + ); - assert!(output.status.success()); + run_command( + TWOLITER_PATH, + [ + "fetch", + "--project-path", + local_kit.join("Twoliter.toml").to_str().unwrap(), + ], + [], + ); - let lock_contents = tokio::fs::read_to_string(&lockfile).await.unwrap(); - assert_eq!(lock_contents, EXPECTED_LOCKFILE); + run_command( + TWOLITER_PATH, + [ + "build", + "kit", + "core-kit", + "--project-path", + local_kit.join("Twoliter.toml").to_str().unwrap(), + ], + [], + ); + + std::fs::write(local_kit.join("Infra.toml"), INFRA_TOML).unwrap(); + run_command( + TWOLITER_PATH, + [ + "publish", + "kit", + "--project-path", + local_kit.join("Twoliter.toml").to_str().unwrap(), + "core-kit", + "bottlerocket", + "core-kit-overridden", + ], + [("SSL_CERT_FILE", registry.cert_file().to_str().unwrap())], + ); + } +} - tokio::fs::remove_file(&lockfile).await.ok(); +impl Drop for LocalKit { + fn drop(&mut self) { + let local_kit = test_projects_dir().join("local-kit"); + std::fs::remove_file(local_kit.join("Twoliter.lock")).ok(); + std::fs::remove_file(local_kit.join("Infra.toml")).ok(); + } } diff --git a/tests/projects/external-kit/.gitignore b/tests/projects/external-kit/.gitignore index 41c75f0c2..fa6f85fe9 100644 --- a/tests/projects/external-kit/.gitignore +++ b/tests/projects/external-kit/.gitignore @@ -8,3 +8,5 @@ Test.toml testsys.kubeconfig Infra.toml +Twoliter.lock +Twoliter.override diff --git a/tests/projects/external-kit/Twoliter.toml b/tests/projects/external-kit/Twoliter.toml index ea47adb8d..fdaf39783 100644 --- a/tests/projects/external-kit/Twoliter.toml +++ b/tests/projects/external-kit/Twoliter.toml @@ -5,9 +5,9 @@ release-version = "1.0.0" registry = "public.ecr.aws/bottlerocket" [vendor.custom-vendor] -registry = "public.ecr.aws/bottlerocket" +registry = "definitely-wont-resolve" [[kit]] -name = "bottlerocket-core-kit" -version = "2.0.0" +name = "core-kit" +version = "1.0.0" vendor = "custom-vendor" diff --git a/tests/projects/local-kit/.gitignore b/tests/projects/local-kit/.gitignore index 41c75f0c2..012a0c13d 100644 --- a/tests/projects/local-kit/.gitignore +++ b/tests/projects/local-kit/.gitignore @@ -8,3 +8,4 @@ Test.toml testsys.kubeconfig Infra.toml +Twoliter.lock diff --git a/tools/attribution/attribution.sh b/tools/attribution/attribution.sh index 8e2a45313..05de4a299 100755 --- a/tools/attribution/attribution.sh +++ b/tools/attribution/attribution.sh @@ -25,6 +25,25 @@ echo "Clarifying crate dependency licenses..." --out-dir ${LICENSEDIR}/vendor \ cargo --locked Cargo.toml +# =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= +# go-containerregistry +pushd /src/tools/krane +../build-cache-fetch hashes/crane +TARBALL=$(grep -oP '\(\K[^\)]*' hashes/crane) +GO_CONTAINERREGISTRY_UNPACK_DIR=$(mktemp -d) +tar --strip-components=1 -xvf "${TARBALL}" -C "${GO_CONTAINERREGISTRY_UNPACK_DIR}" + +pushd "${GO_CONTAINERREGISTRY_UNPACK_DIR}/cmd/krane" +go mod vendor +popd + +/usr/libexec/tools/bottlerocket-license-scan \ + --clarify /src/clarify.toml \ + --spdx-data /usr/libexec/tools/spdx-data \ + --out-dir ${LICENSEDIR}/krane \ + go-vendor "${GO_CONTAINERREGISTRY_UNPACK_DIR}/cmd/krane/vendor" +popd + # =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= =^.^= # cargo-make (we currently use cargo-make from the SDK, but will ship it in Twoliter in the future) echo "Clarifying bottlerocket-sdk & dependency licenses..." @@ -37,6 +56,6 @@ cp -r /usr/share/licenses/cargo-make \ cp /src/COPYRIGHT /src/LICENSE-MIT /src/LICENSE-APACHE \ ${LICENSEDIR}/ -pushd $(dirname ${LICENSEDIR}) -tar czf /src/twoliter-attributions.tar.gz $(basename ${LICENSEDIR}) +pushd "$(dirname ${LICENSEDIR})" +tar czf /src/twoliter-attributions.tar.gz "$(basename ${LICENSEDIR})" popd diff --git a/tools/build-cache-fetch b/tools/build-cache-fetch new file mode 100755 index 000000000..0b6539158 --- /dev/null +++ b/tools/build-cache-fetch @@ -0,0 +1,15 @@ +#!/bin/bash +# Fetches sources from Bottlerocket's build cache, cache.bottlerocket.aws + +set -euxo pipefail +UPSTREAM_SOURCE_FALLBACK="${UPSTREAM_SOURCE_FALLBACK:-'false'}" +# shellcheck disable=SC2046 +if ! curl --fail --remote-name-all --remote-time \ + $(awk -F '[ ()]' '/^SHA512 \(/ { + printf "https://cache.bottlerocket.aws/%s/%s/%s\n", $3, $6, $3 + }' "$1") \ +&& [[ "${UPSTREAM_SOURCE_FALLBACK}" == 'true' ]]; then + curl --fail --remote-name-all --remote-time --location \ + $(awk '/^#\s((s)?ftp|http(s)?):\/\// {printf "%s\n", $2}' "$1") +fi +sha512sum --check "$1" diff --git a/tools/krane/Cargo.toml b/tools/krane/Cargo.toml new file mode 100644 index 000000000..20879f78f --- /dev/null +++ b/tools/krane/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "krane-bundle" +version = "0.1.0" +authors = ["Sean P. Kelly "] +license = "Apache-2.0 OR MIT" +edition = "2021" +publish = false + +[dependencies] +anyhow.workspace = true +flate2.workspace = true +lazy_static.workspace = true +pentacle.workspace = true + +[build-dependencies] +flate2.workspace = true +tar.workspace = true diff --git a/tools/krane/README.md b/tools/krane/README.md new file mode 100644 index 000000000..076b7cacc --- /dev/null +++ b/tools/krane/README.md @@ -0,0 +1,11 @@ +## krane-bundle + +This crate packages the `krane` utility from [google/go-containerregistry]. + +The utility is compiled by a build script, the output of which is compressed and stored in the Rust +crate as via `include_bytes!`. +At runtime, `krane-bundle` writes the decompressed binary to a [sealed anonymous file], passing the +filepath of that file to any caller. + +[google/go-containerregistry]: https://github.com/google/go-containerregistry +[sealed anonymous file]: https://github.com/haha-business/pentacle diff --git a/tools/krane/build.rs b/tools/krane/build.rs new file mode 100644 index 000000000..323d08c1b --- /dev/null +++ b/tools/krane/build.rs @@ -0,0 +1,100 @@ +use flate2::{read::GzDecoder, write::GzEncoder}; +use std::env; +use std::fs::File; +use std::io::{self, prelude::*}; +use std::path::PathBuf; +use std::process::Command; +use tar::Archive; + +const CRANE_VERSION: &str = "0.20.1"; + +fn main() { + let script_dir = env::current_dir().unwrap(); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + + println!("cargo::rerun-if-changed=../build-cache-fetch"); + println!("cargo::rerun-if-changed=hashes/crane"); + + // Download and checksum-verify crane + env::set_current_dir(&out_dir).expect("Failed to set current directory"); + Command::new(script_dir.join("../build-cache-fetch")) + .arg(script_dir.join("hashes/crane")) + .status() + .expect("Failed to execute build-cache-fetch"); + + // extract crane sources + let crane_archive = out_dir.join(format!("go-containerregistry-v{CRANE_VERSION}.tar.gz")); + let crane_tgz = File::open(&crane_archive).expect("Failed to open crane archive"); + let mut tar_archive = Archive::new(GzDecoder::new(crane_tgz)); + + let crane_output_dir = out_dir.join(format!("go-containerregistry-v{CRANE_VERSION}")); + tar_archive + .unpack(&crane_output_dir) + .expect("Failed to extract crane sources"); + + // build krane + let build_output_loc = out_dir.join("krane"); + Command::new("go") + .arg("build") + .env("GOOS", get_goos()) + .env("GOARCH", get_goarch()) + .arg("-o") + .arg(&build_output_loc) + .current_dir( + crane_output_dir.join(format!("go-containerregistry-{CRANE_VERSION}/cmd/krane")), + ) + .status() + .expect("Failed to build crane"); + + // compress krane + let krane_gz_path = out_dir.join("krane.gz"); + let compressed_output_file = + File::create(&krane_gz_path).expect("Failed to crate krane.gz file"); + + let krane_binary = File::open(&build_output_loc).expect("Failed to open krane binary"); + let mut reader = io::BufReader::new(&krane_binary); + let mut encoder = GzEncoder::new(&compressed_output_file, flate2::Compression::best()); + + let mut buffer = Vec::with_capacity( + krane_binary + .metadata() + .expect("Failed to get krane binary metadata") + .len() as usize, + ); + reader + .read_to_end(&mut buffer) + .expect("Failed to read krane binary"); + encoder + .write_all(&buffer) + .expect("Failed to write compressed krane binary"); + encoder + .finish() + .expect("Failed to finish writing compressed krane binary"); + + println!("cargo::rustc-env=KRANE_GZ_PATH={}", krane_gz_path.display()); +} + +fn get_goos() -> &'static str { + let target_os = env::var("CARGO_CFG_TARGET_OS").expect("Failed to read CARGO_CFG_TARGET_OS"); + match target_os.as_str() { + "linux" => "linux", + "windows" => "windows", + "macos" => "darwin", + // Add more mappings as needed + other => panic!("Unsupported target OS: {}", other), + } +} + +fn get_goarch() -> &'static str { + let target_arch = + env::var("CARGO_CFG_TARGET_ARCH").expect("Failed to read CARGO_CFG_TARGET_ARCH"); + + match target_arch.as_str() { + "x86_64" => "amd64", + "aarch64" => "arm64", + "arm" => "arm", + "wasm32" => "wasm", + // Add more mappings as needed + other => panic!("Unsupported target architecture: {}", other), + } +} diff --git a/tools/krane/hashes/crane b/tools/krane/hashes/crane new file mode 100644 index 000000000..a0f357326 --- /dev/null +++ b/tools/krane/hashes/crane @@ -0,0 +1,2 @@ +# https://github.com/google/go-containerregistry/archive/refs/tags/v0.20.1.tar.gz +SHA512 (go-containerregistry-v0.20.1.tar.gz) = c323c5b78c35fb7af67641fa4ef1802b944f8bd908163ff40a952a0c190e2dd210100efba1fbc2064495cff28a60d5bc7ee98e510ec116522c7897539b02fad8 diff --git a/tools/krane/src/lib.rs b/tools/krane/src/lib.rs new file mode 100644 index 000000000..171583870 --- /dev/null +++ b/tools/krane/src/lib.rs @@ -0,0 +1,61 @@ +use anyhow::{Context, Result}; +use flate2::read::GzDecoder; +use std::fs::File; +use std::os::fd::AsRawFd; +use std::path::{Path, PathBuf}; + +use pentacle::SealOptions; + +const COMPRESSED_KRANE_BIN: &[u8] = include_bytes!(env!("KRANE_GZ_PATH")); + +lazy_static::lazy_static! { + pub static ref KRANE: Krane = Krane::seal().unwrap(); +} + +#[derive(Debug)] +pub struct Krane { + // Hold the file in memory to keep the fd open + _sealed_binary: File, + path: PathBuf, +} + +impl Krane { + fn seal() -> Result { + let mut krane_reader = GzDecoder::new(COMPRESSED_KRANE_BIN); + + let sealed_binary = SealOptions::new() + .close_on_exec(false) + .executable(true) + .copy_and_seal(&mut krane_reader) + .context("Failed to write krane binary to sealed anonymous file")?; + + let fd = sealed_binary.as_raw_fd(); + let pid = std::process::id(); + let path = PathBuf::from(format!("/proc/{pid}/fd/{fd}")); + + Ok(Krane { + _sealed_binary: sealed_binary, + path, + }) + } + + pub fn path(&self) -> &Path { + &self.path + } +} + +#[cfg(test)] +mod test { + use super::*; + use std::process::Command; + + #[test] + fn test_krane_runs() { + let status = Command::new(KRANE.path()) + .arg("--help") + .output() + .expect("failed to run krane"); + + assert_eq!(status.status.code().unwrap(), 0); + } +} diff --git a/tools/oci-cli-wrapper/Cargo.toml b/tools/oci-cli-wrapper/Cargo.toml index dcbd2f833..9f1d56f45 100644 --- a/tools/oci-cli-wrapper/Cargo.toml +++ b/tools/oci-cli-wrapper/Cargo.toml @@ -8,6 +8,7 @@ publish = false [dependencies] async-trait.workspace = true +krane-bundle.workspace = true log.workspace = true olpc-cjson.workspace = true regex.workspace = true diff --git a/tools/oci-cli-wrapper/src/docker.rs b/tools/oci-cli-wrapper/src/docker.rs deleted file mode 100644 index f2cf140e7..000000000 --- a/tools/oci-cli-wrapper/src/docker.rs +++ /dev/null @@ -1,148 +0,0 @@ -use std::path::Path; - -use async_trait::async_trait; -use regex::Regex; -use snafu::{OptionExt, ResultExt}; -use std::fs::File; -use tar::Archive; -use tempfile::NamedTempFile; - -use crate::cli::CommandLine; -use crate::{error, ConfigView, DockerArchitecture, ImageToolImpl, Result}; - -#[derive(Debug)] -pub struct DockerCLI { - pub(crate) cli: CommandLine, -} - -#[async_trait] -impl ImageToolImpl for DockerCLI { - async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()> { - // First we pull the image to local daemon - self.cli - .spawn( - &["pull", uri], - format!("failed to pull image to local docker from {}", uri), - ) - .await?; - // Now we can use docker save to save the archive to a temppath - let temp_file = NamedTempFile::new_in(path).context(crate::error::DockerTempSnafu)?; - let tmp_path = temp_file.path().to_string_lossy(); - self.cli - .spawn( - &["save", uri, "-o", tmp_path.as_ref()], - format!("failed to save image archive from {} to {}", uri, tmp_path), - ) - .await?; - let archive_file = File::open(temp_file.path()).context(crate::error::ArchiveReadSnafu)?; - let mut archive = Archive::new(archive_file); - archive - .unpack(path) - .context(crate::error::ArchiveExtractSnafu)?; - Ok(()) - } - - async fn get_manifest(&self, uri: &str) -> Result> { - self.cli - .output( - &["manifest", "inspect", uri], - format!("failed to inspect manifest of resource at {}", uri), - ) - .await - } - - async fn get_config(&self, uri: &str) -> Result { - self.cli - .spawn(&["pull", uri], format!("failed to pull image from {}", uri)) - .await?; - let bytes = self - .cli - .output( - &["image", "inspect", uri, "--format", "{{ json .Config }}"], - format!("failed to fetch image config from {}", uri), - ) - .await?; - serde_json::from_slice(bytes.as_slice()).context(error::ConfigDeserializeSnafu) - } - - async fn push_oci_archive(&self, path: &Path, uri: &str) -> Result<()> { - let out = self - .cli - .output( - &["load", format!("--input={}", path.display()).as_str()], - format!("could not load archive from {}", path.display()), - ) - .await?; - let out = String::from_utf8_lossy(&out); - let digest_expression = - Regex::new("(?sha256:[0-9a-f]{64})").context(error::RegexSnafu)?; - let caps = digest_expression - .captures(&out) - .context(error::NoDigestSnafu)?; - let digest = &caps["digest"]; - - self.cli - .output( - &["tag", digest, uri], - format!("could not tag image as {uri}"), - ) - .await?; - - self.cli - .spawn(&["push", uri], format!("failed to push image '{uri}'")) - .await?; - - Ok(()) - } - - async fn push_multi_platform_manifest( - &self, - platform_images: Vec<(DockerArchitecture, String)>, - uri: &str, - ) -> Result<()> { - let images: Vec<&str> = platform_images - .iter() - .map(|(_, image)| image.as_str()) - .collect(); - - let mut manifest_create_args = vec!["manifest", "create", uri]; - manifest_create_args.extend_from_slice(&images); - self.cli - .output( - &manifest_create_args, - format!("could not create manifest list {uri}"), - ) - .await?; - - for (arch, image) in platform_images.iter() { - self.cli - .output( - &[ - "manifest", - "annotate", - format!("--arch={}", arch).as_str(), - uri, - image, - ], - format!("could not annotate manifest {uri} for arch {arch}"), - ) - .await?; - } - - self.cli - .output( - &["manifest", "push", uri], - format!("could not push manifest to {uri}"), - ) - .await?; - - self.cli - .output( - &["manifest", "rm", uri], - format!("could not delete manifest {uri}"), - ) - .await?; - - Ok(()) - } -} diff --git a/tools/oci-cli-wrapper/src/lib.rs b/tools/oci-cli-wrapper/src/lib.rs index 81a64691b..1949543f9 100644 --- a/tools/oci-cli-wrapper/src/lib.rs +++ b/tools/oci-cli-wrapper/src/lib.rs @@ -12,20 +12,18 @@ //! metadata. In addition, in order to operate with OCI image format, the containerd-snapshotter //! feature has to be enabled in the docker daemon use std::fmt::{Display, Formatter}; -use std::{collections::HashMap, env, path::Path}; +use std::{collections::HashMap, path::Path}; use async_trait::async_trait; use cli::CommandLine; use crane::CraneCLI; -use docker::DockerCLI; +use krane_bundle::KRANE; use olpc_cjson::CanonicalFormatter; use serde::{Deserialize, Serialize}; use snafu::ResultExt; -use which::which; mod cli; mod crane; -mod docker; #[derive(Debug)] pub struct ImageTool { @@ -33,62 +31,14 @@ pub struct ImageTool { } impl ImageTool { - /// Uses the container tool specified by the given tool name. - /// - /// The specified tool must be present in the unix search path. - fn from_tool_name(tool_name: &str) -> Result { - let image_tool_impl: Box = match tool_name { - "docker" => Box::new(DockerCLI { - cli: CommandLine { - path: which("docker").context(error::NotFoundSnafu { name: "docker" })?, - }, - }), - tool @ ("crane" | "gcrane" | "krane") => Box::new(CraneCLI { - cli: CommandLine { - path: which(tool).context(error::NotFoundSnafu { name: tool })?, - }, - }), - _ => return error::UnsupportedSnafu { name: tool_name }.fail(), - }; - - Ok(Self { image_tool_impl }) - } - - /// Auto-selects the container tool based on unix search path. - /// - /// Uses `crane` if available, falling back to `docker` otherwise. - fn from_unix_search_path() -> Result { - let crane = which("krane").or(which("gcrane")).or(which("crane")); - let image_tool_impl: Box = if let Ok(path) = crane { - Box::new(CraneCLI { - cli: CommandLine { path }, - }) - } else { - Box::new(DockerCLI { - cli: CommandLine { - path: which("docker").context(error::NoneFoundSnafu)?, - }, - }) - }; - - Ok(Self { image_tool_impl }) - } - - /// Auto-select the container tool to use by environment variable - /// and-or auto detection. - /// - /// If TWOLITER_KIT_IMAGE_TOOL environment variable is set, uses that value. - /// Valid values are: - /// * docker - /// * crane | gcrane | krane - /// - /// Otherwise, searches $PATH, using `crane` if available and falling back to docker otherwise. - pub fn from_environment() -> Result { - if let Ok(name) = env::var("TWOLITER_KIT_IMAGE_TOOL") { - Self::from_tool_name(&name) - } else { - Self::from_unix_search_path() - } + /// Uses the builtin `krane` provided by the `tools/krane` crate. + pub fn from_builtin_krane() -> Self { + let image_tool_impl = Box::new(CraneCLI { + cli: CommandLine { + path: KRANE.path().to_path_buf(), + }, + }); + Self { image_tool_impl } } pub fn new(image_tool_impl: Box) -> Self { diff --git a/tools/pubsys/src/kit/publish_kit/mod.rs b/tools/pubsys/src/kit/publish_kit/mod.rs index 9dd5ffbab..07aff00a5 100644 --- a/tools/pubsys/src/kit/publish_kit/mod.rs +++ b/tools/pubsys/src/kit/publish_kit/mod.rs @@ -31,7 +31,7 @@ pub(crate) struct PublishKitArgs { } pub(crate) async fn run(args: &Args, publish_kit_args: &PublishKitArgs) -> Result<()> { - let image_tool = ImageTool::from_environment().context(error::ImageToolSnafu)?; + let image_tool = ImageTool::from_builtin_krane(); // If a lock file exists, use that, otherwise use Infra.toml let infra_config = InfraConfig::from_path_or_lock(&args.infra_config_path, false) @@ -137,11 +137,6 @@ mod error { #[snafu(display("Error reading config: {}", source))] Config { source: pubsys_config::Error }, - #[snafu(display("Could not find image tool: {}", source))] - ImageTool { - source: oci_cli_wrapper::error::Error, - }, - #[snafu(display("Could not convert {} to docker architecture: {}", arch, source))] InvalidArchitecture { source: oci_cli_wrapper::error::Error, diff --git a/tools/unplug/Cargo.toml b/tools/unplug/Cargo.toml index b07cd7d7f..dc0313403 100644 --- a/tools/unplug/Cargo.toml +++ b/tools/unplug/Cargo.toml @@ -10,5 +10,5 @@ publish = false anyhow.workspace = true [target.'cfg(target_os = "linux")'.dependencies] -libc = "0.2.148" -seccompiler = "0.4.0" +libc.workspace = true +seccompiler.workspace = true diff --git a/twoliter/embedded/rpm2kit b/twoliter/embedded/rpm2kit index 24aab8a85..8775f435b 100755 --- a/twoliter/embedded/rpm2kit +++ b/twoliter/embedded/rpm2kit @@ -125,7 +125,7 @@ CONFIG="$(jq --compact-output < String { + format!("{KIT_METADATA_LABEL_PREFIX}{SUPPORTED_KIT_METADATA_VERSION}") +} + /// Represents a locked dependency on an image #[derive(Debug, Clone, Eq, Ord, PartialOrd, Serialize, Deserialize)] pub(crate) struct LockedImage { @@ -59,12 +69,13 @@ impl VendedArtifact for LockedImage { } #[derive(Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] pub(crate) struct ImageMetadata { /// The name of the kit - #[allow(dead_code)] + #[expect(dead_code)] pub name: String, /// The version of the kit - #[allow(dead_code)] + #[expect(dead_code)] pub version: Version, /// The required sdk of the kit, pub sdk: Image, @@ -93,13 +104,7 @@ impl EncodedKitMetadata { async fn try_from_image(image_uri: &str, image_tool: &ImageTool) -> Result { tracing::trace!(image_uri, "Extracting kit metadata from OCI image config"); let config = image_tool.get_config(image_uri).await?; - let kit_metadata = EncodedKitMetadata( - config - .labels - .get("dev.bottlerocket.kit.v1") - .context("no metadata stored on image, this image appears to not be a kit")? - .to_owned(), - ); + let kit_metadata = EncodedKitMetadata(Self::extract_encoded_kit_metadata(&config)?); tracing::trace!( image_uri, @@ -111,6 +116,53 @@ impl EncodedKitMetadata { Ok(kit_metadata) } + fn extract_encoded_kit_metadata(oci_config: &ConfigView) -> Result { + let encoded_metadata = oci_config + .labels + .get(supported_kit_metadata_label().as_str()); + + match encoded_metadata { + Some(encoded_metadata) => Ok(encoded_metadata.to_owned()), + None => { + if let Some(kit_label) = oci_config + .labels + .keys() + .find(|label| label.starts_with(KIT_METADATA_LABEL_PREFIX)) + { + let kit_version = kit_label.trim_start_matches(KIT_METADATA_LABEL_PREFIX); + let meta_relation = + Self::compare_version_strs(kit_version, SUPPORTED_KIT_METADATA_VERSION); + + bail!( + "kit appears to be built with metadata version '{kit_version}', possibly by \ + {meta_relation} version of twoliter with unsupported incompatibilities. \ + This version of twoliter supports metadata version \ + '{SUPPORTED_KIT_METADATA_VERSION}'.", + ) + } else { + bail!("no metadata stored on image, this image appears not to be a kit") + } + } + } + } + + /// Compare's kit metadata versions in english. Intended to be used in error messages. + fn compare_version_strs(lhs: &str, rhs: &str) -> &'static str { + let lhs: Result = lhs.trim_start_matches('v').parse(); + let rhs = rhs.trim_start_matches('v').parse(); + + match (lhs, rhs) { + (Ok(lhs), Ok(rhs)) => { + if lhs < rhs { + "an older" + } else { + "a newer" + } + } + _ => "a different", + } + } + /// Infallible method to provide debugging insights into encoded `ImageMetadata` /// /// Shows a `Debug` view of the encoded `ImageMetadata` if possible, otherwise shows @@ -311,6 +363,8 @@ impl ImageResolver { #[cfg(test)] mod test { use super::*; + use std::collections::HashMap; + #[test] fn test_try_debug_image_metadata_succeeds() { // Given a valid encoded metadata string, @@ -334,4 +388,65 @@ mod test { let junk_data = EncodedKitMetadata("abcdefghijklmnophello".to_string()); assert!(junk_data.debug_image_metadata().is_none()); } + + #[test] + fn test_extract_encoded_kit_metadata_fails_no_label() { + EncodedKitMetadata::extract_encoded_kit_metadata(&ConfigView { + labels: HashMap::from([("foo".to_string(), "bar".to_string())]), + }) + .expect_err("no label"); + } + + #[test] + fn test_extract_encoded_kit_metadata_fails_older_metadata() { + let err = EncodedKitMetadata::extract_encoded_kit_metadata(&ConfigView { + labels: HashMap::from([(format!("{KIT_METADATA_LABEL_PREFIX}v0"), "bar".to_string())]), + }) + .expect_err("too old") + .to_string(); + + assert!(err.contains("older") && err.contains("incompatibilities")); + } + + #[test] + fn test_extract_encoded_kit_metadata_fails_newer_metadata() { + let err = EncodedKitMetadata::extract_encoded_kit_metadata(&ConfigView { + labels: HashMap::from([( + format!("{KIT_METADATA_LABEL_PREFIX}v9999"), + "bar".to_string(), + )]), + }) + .expect_err("too new") + .to_string(); + + assert!(err.contains("newer") && err.contains("incompatibilities")); + } + + #[test] + fn test_extract_encoded_kit_metadata_fails_metadata_ver_unparseable() { + let err = EncodedKitMetadata::extract_encoded_kit_metadata(&ConfigView { + labels: HashMap::from([( + format!("{KIT_METADATA_LABEL_PREFIX}notaversion"), + "foo".to_string(), + )]), + }) + .expect_err("not a version") + .to_string(); + + assert!(err.contains("different") && err.contains("incompatibilities")); + } + + #[test] + fn test_extract_encoded_kit_metadata_succeeds_current_metadata_version() { + assert_eq!( + EncodedKitMetadata::extract_encoded_kit_metadata(&ConfigView { + labels: HashMap::from([( + format!("{KIT_METADATA_LABEL_PREFIX}{SUPPORTED_KIT_METADATA_VERSION}"), + "bar".to_string(), + )]), + }) + .unwrap(), + "bar".to_string() + ); + } } diff --git a/twoliter/src/project/lock/mod.rs b/twoliter/src/project/lock/mod.rs index e52497ae4..6d9a07b99 100644 --- a/twoliter/src/project/lock/mod.rs +++ b/twoliter/src/project/lock/mod.rs @@ -103,7 +103,7 @@ impl LockedSDK { }; debug!(?sdk, "Resolving workspace SDK"); - let image_tool = ImageTool::from_environment()?; + let image_tool = ImageTool::from_builtin_krane(); ImageResolver::from_image(&sdk)? .skip_metadata_retrieval() // SDKs don't have metadata .resolve(&image_tool) @@ -203,7 +203,7 @@ impl Lock { /// Fetches all external kits defined in a Twoliter.lock to the build directory #[instrument(level = "trace", skip_all)] pub(crate) async fn fetch(&self, project: &Project, arch: &str) -> Result<()> { - let image_tool = ImageTool::from_environment()?; + let image_tool = ImageTool::from_builtin_krane(); let target_dir = project.external_kits_dir(); create_dir_all(&target_dir).await.context(format!( "failed to create external-kits directory at {}", @@ -257,7 +257,7 @@ impl Lock { async fn resolve(project: &Project) -> Result { let mut known: HashMap<(ValidIdentifier, ValidIdentifier), Version> = HashMap::new(); let mut locked: Vec = Vec::new(); - let image_tool = ImageTool::from_environment()?; + let image_tool = ImageTool::from_builtin_krane(); let mut remaining = project.direct_kit_deps()?; let mut sdk_set = HashSet::new(); diff --git a/twoliter/src/project/mod.rs b/twoliter/src/project/mod.rs index 6c7889473..635341b71 100644 --- a/twoliter/src/project/mod.rs +++ b/twoliter/src/project/mod.rs @@ -6,6 +6,7 @@ pub(crate) use lock::VerificationTagger; use self::lock::{Lock, LockedSDK, Override}; use crate::common::fs::{self, read_to_string}; +use crate::compatibility::SUPPORTED_TWOLITER_PROJECT_SCHEMA_VERSION; use crate::docker::ImageUri; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; @@ -51,7 +52,7 @@ pub(crate) struct Project { project_dir: PathBuf, /// The version of this schema struct. - schema_version: SchemaVersion<1>, + schema_version: SchemaVersion, /// The version that will be given to released artifacts such as kits and variants. release_version: String,