diff --git a/.ci/ci_job_flags.sh b/.ci/ci_job_flags.sh index 851215805..5ae67c924 100755 --- a/.ci/ci_job_flags.sh +++ b/.ci/ci_job_flags.sh @@ -40,6 +40,13 @@ init_ci_flags() { # Use the forked version of containerd for Confidential Containers # Valyes: "yes|no" export FORKED_CONTAINERD="no" + # Do the pull on the guest using the upstream containerd for Confidential Containers + # Values: "yes|no" + export IMAGE_OFFLOAD_TO_GUEST="no" + # Do the pull on the host using the upstream containerd for Confidential Containers + # Values: "yes|no" + export PULL_ON_HOST="no" + export PULL_ON_HOST_EXPORT_MODE="image_block" # Hypervisor to use export KATA_HYPERVISOR="" # Install k8s @@ -131,6 +138,50 @@ case "${CI_JOB}" in ;; esac ;; +"CC_CRI_CONTAINERD_K8S_IMAGE_OFFLOAD_TO_GUEST") + # This job only tests containerd + k8s + init_ci_flags + export CRI_CONTAINERD="yes" + export CRI_RUNTIME="containerd" + export KATA_HYPERVISOR="qemu" + export KUBERNETES="yes" + # Export any CC specific environment variables + export KATA_BUILD_CC="yes" + export FORKED_CONTAINERD="no" + export IMAGE_OFFLOAD_TO_GUEST="yes" + export MEASURED_ROOTFS="yes" + export AA_KBC="offline_fs_kbc" + ;; +"CC_CRI_CONTAINERD_K8S_PULL_ON_HOST_IMAGE_BLOCK") + # This job only tests containerd + k8s + init_ci_flags + export CRI_CONTAINERD="yes" + export CRI_RUNTIME="containerd" + export KATA_HYPERVISOR="qemu" + export KUBERNETES="yes" + # Export any CC specific environment variables + export KATA_BUILD_CC="yes" + export FORKED_CONTAINERD="no" + export PULL_ON_HOST="yes" + export PULL_ON_HOST_EXPORT_MODE="image_block" + export MEASURED_ROOTFS="yes" + export AA_KBC="offline_fs_kbc" + ;; +"CC_CRI_CONTAINERD_K8S_PULL_ON_HOST_IMAGE_BLOCK_WITH_VERITY") + # This job only tests containerd + k8s + init_ci_flags + export CRI_CONTAINERD="yes" + export CRI_RUNTIME="containerd" + export KATA_HYPERVISOR="qemu" + export KUBERNETES="yes" + # Export any CC specific environment variables + export KATA_BUILD_CC="yes" + export FORKED_CONTAINERD="no" + export PULL_ON_HOST="yes" + export PULL_ON_HOST_EXPORT_MODE="image_block_with_verity" + export MEASURED_ROOTFS="yes" + export AA_KBC="offline_fs_kbc" + ;; "CC_SEV_CRI_CONTAINERD_K8S"|"CC_SNP_CRI_CONTAINERD_K8S") init_ci_flags export CRI_CONTAINERD="yes" diff --git a/.ci/containerd_nydus_setup.sh b/.ci/containerd_nydus_setup.sh new file mode 100755 index 000000000..1173771ad --- /dev/null +++ b/.ci/containerd_nydus_setup.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail + +cidir=$(dirname "$0") +source "${cidir}/../lib/common.bash" + +# Nydus related configurations +NYDUS_SNAPSHOTTER_BINARY="/usr/local/bin/containerd-nydus-grpc" +NYDUS_SNAPSHOTTER_TARFS_CONFIG="/usr/local/share/nydus-snapshotter/config-coco-host-sharing.toml" +NYDUS_SNAPSHOTTER_GUEST_CONFIG="/usr/local/share/nydus-snapshotter/config-coco-guest-pulling.toml" +NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_CONFIG:-${NYDUS_SNAPSHOTTER_TARFS_CONFIG}}" +NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE="${PULL_ON_HOST_EXPORT_MODE:-image_block}" + +echo "Configure nydus snapshotter" +if [ "${IMAGE_OFFLOAD_TO_GUEST:-"no"}" == "yes" ]; then + echo "Pulling image on the guest" + NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_GUEST_CONFIG}" +else + echo "Pulling image on the host | export_mode = ${NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE}" + NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_TARFS_CONFIG}" + sudo sed -i "s/export_mode = .*/export_mode = \"${NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE}\"/" "$NYDUS_SNAPSHOTTER_CONFIG" +fi + +echo "Start nydus snapshotter" +systemd-cat -t snapshotter sudo "${NYDUS_SNAPSHOTTER_BINARY}" --config "${NYDUS_SNAPSHOTTER_CONFIG}" --log-to-stdout --log-level debug & + +echo "Configure containerd to use the nydus snapshotter" + +containerd_config_file="/etc/containerd/config.toml" + +snapshotter_socket="/run/containerd-nydus/containerd-nydus-grpc.sock" +proxy_config=" [proxy_plugins.nydus]\n type = \"snapshot\"\n address = \"${snapshotter_socket}\"" +snapshotter_config=" disable_snapshot_annotations = false\n snapshotter = \"nydus\"" + +echo -e "[proxy_plugins]" | sudo tee -a "${containerd_config_file}" +echo -e "${proxy_config}" | sudo tee -a "${containerd_config_file}" + +sudo sed -i '/\[plugins.cri.containerd\]/a\'"${snapshotter_config}" "${containerd_config_file}" +sudo systemctl restart containerd diff --git a/.ci/install_nydus_snapshotter.sh b/.ci/install_nydus_snapshotter.sh new file mode 100755 index 000000000..469734c4e --- /dev/null +++ b/.ci/install_nydus_snapshotter.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# +# Copyright (c) 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +set -o errexit +set -o nounset +set -o pipefail +set -o errtrace + +cidir=$(dirname "$0") +source "${cidir}/lib.sh" + +target_dir="/usr/local/" +nydus_snapshotter_repo=$(get_version "externals.nydus-snapshotter.url") +nydus_snapshotter_version=$(get_version "externals.nydus-snapshotter.version") +nydus_snapshotter_repo_dir="${GOPATH}/src/github.com/containerd/nydus-snapshotter" +nydus_snapshotter_binary_target_dir="${target_dir}/bin" +nydus_snapshotter_config_target_dir="${target_dir}/share/nydus-snapshotter" + +nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"} +nydus_version=${nydus_version:-"v2.3.0-alpha.0"} + +arch="$(uname -m)" + +clone_nydus_snapshotter_repo() { + add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}" + + if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then + mkdir -p "${nydus_snapshotter_repo_dir}" + git clone ${nydus_snapshotter_repo} "${nydus_snapshotter_repo_dir}" || true + pushd "${nydus_snapshotter_repo_dir}" + git checkout "${nydus_snapshotter_version}" + popd + fi +} + +build_nydus_snapshotter() { + pushd "${nydus_snapshotter_repo_dir}" + if [ "${arch}" = "s390x" ]; then + export GOARCH=${arch} + fi + make + + sudo install -D -m 755 "bin/containerd-nydus-grpc" "${nydus_snapshotter_binary_target_dir}/containerd-nydus-grpc" + sudo install -D -m 755 "bin/nydus-overlayfs" "${nydus_snapshotter_binary_target_dir}/nydus-overlayfs" + rm -rf "${nydus_snapshotter_repo_dir}/bin" + popd >/dev/null +} + +download_nydus_snapshotter_config() { + tmp_dir=$(mktemp -d -t install-nydus-snapshotter-config-tmp.XXXXXXXXXX) + curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/${nydus_snapshotter_version}/misc/snapshotter/config-coco-guest-pulling.toml -o "${tmp_dir}/config-coco-guest-pulling.toml" + curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/${nydus_snapshotter_version}/misc/snapshotter/config-coco-host-sharing.toml -o "${tmp_dir}/config-coco-host-sharing.toml" + sudo install -D -m 644 "${tmp_dir}/config-coco-guest-pulling.toml" "${nydus_snapshotter_config_target_dir}/config-coco-guest-pulling.toml" + sudo install -D -m 644 "${tmp_dir}/config-coco-host-sharing.toml" "${nydus_snapshotter_config_target_dir}/config-coco-host-sharing.toml" + +} + +download_nydus_from_tarball() { + if [ "${arch}" = "s390x" ]; then + echo "Skip to download nydus for ${arch}, it doesn't work for ${arch} now." + return + fi + + local goarch="$(${cidir}/kata-arch.sh --golang)" + local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-${goarch}.tgz" + echo "Download tarball from ${tarball_url}" + tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX) + curl -Ls "$tarball_url" | sudo tar xfz - -C ${tmp_dir} --strip-components=1 + sudo install -D -m 755 "${tmp_dir}/nydus-image" "${target_dir}/bin/" +} + +download_nydus_from_tarball +clone_nydus_snapshotter_repo +build_nydus_snapshotter +download_nydus_snapshotter_config +echo "install nydus-snapshotter successful" diff --git a/.ci/run.sh b/.ci/run.sh index 8e68f971c..371ac9ce8 100755 --- a/.ci/run.sh +++ b/.ci/run.sh @@ -85,7 +85,19 @@ case "${CI_JOB}" in ;; "CC_CRI_CONTAINERD_K8S"|"CC_CRI_CONTAINERD_K8S_TDX_QEMU"|"CC_CRI_CONTAINERD_K8S_SE_QEMU"|"CC_CRI_CONTAINERD_K8S_TDX_CLOUD_HYPERVISOR") info "Running Confidential Container tests" - sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-kubernetes" + sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="no" PULL_ON_HOST="no" FORKED_CONTAINERD="yes" bash -c "make cc-kubernetes" + ;; + "CC_CRI_CONTAINERD_K8S_IMAGE_OFFLOAD_TO_GUEST") + info "Running Confidential Container tests using nydus to offload the image pulling to the guest" + sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="yes" PULL_ON_HOST="no" bash -c "make cc-kubernetes" + ;; + "CC_CRI_CONTAINERD_K8S_PULL_ON_HOST_IMAGE_BLOCK") + info "Running Confidential Container tests using nydus to offload the image pulling to the guest" + sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="no" PULL_ON_HOST="yes" PULL_ON_HOST_EXPORT_MODE="image_block" FORKED_CONTAINERD="no" bash -c "make cc-kubernetes" + ;; + "CC_CRI_CONTAINERD_K8S_PULL_ON_HOST_IMAGE_BLOCK_WITH_VERITY") + info "Running Confidential Container tests using nydus to offload the image pulling to the guest" + sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="no" PULL_ON_HOST="yes" PULL_ON_HOST_EXPORT_MODE="image_block_with_verity" FORKED_CONTAINERD="no" bash -c "make cc-kubernetes" ;; "CRIO_K8S") echo "INFO: Running kubernetes tests" diff --git a/integration/confidential/lib.sh b/integration/confidential/lib.sh index 6e0873d05..4fd4e33c8 100644 --- a/integration/confidential/lib.sh +++ b/integration/confidential/lib.sh @@ -209,11 +209,14 @@ configure_cc_containerd() { waitForProcess 30 5 "sudo crictl info >/dev/null" # Ensure the cc CRI handler is set. - local cri_handler=$(sudo crictl info | \ - jq '.config.containerd.runtimes.kata.cri_handler') - if [[ ! "$cri_handler" =~ cc ]]; then - sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \ - "$containerd_conf_file" + + if [ "${FORKED_CONTAINERD:-no}" = "yes" ]; then + local cri_handler=$(sudo crictl info | \ + jq '.config.containerd.runtimes.kata.cri_handler') + if [[ ! "$cri_handler" =~ cc ]]; then + sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \ + "$containerd_conf_file" + fi fi if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then diff --git a/integration/kubernetes/cleanup_env.sh b/integration/kubernetes/cleanup_env.sh index cf7b0f093..a227b18c8 100755 --- a/integration/kubernetes/cleanup_env.sh +++ b/integration/kubernetes/cleanup_env.sh @@ -40,6 +40,10 @@ main () { info "Stop ${CRI_RUNTIME} service" sudo systemctl stop "${CRI_RUNTIME}" + sudo kill -9 $(pidof "containerd-nydus-grpc") || true + sudo rm -f "/usr/local/bin/nydus-overlayfs" + sudo rm -f "/usr/local/bin/nydus-image" + info "Remove network devices" for dev in cni0 flannel.1; do info "remove device: $dev" diff --git a/integration/kubernetes/confidential/agent_image.bats b/integration/kubernetes/confidential/agent_image.bats index de5c815e8..415d3e355 100644 --- a/integration/kubernetes/confidential/agent_image.bats +++ b/integration/kubernetes/confidential/agent_image.bats @@ -32,6 +32,7 @@ test_tag="[cc][agent][kubernetes][containerd]" setup() { setup_common + switch_image_service_offload off } @test "$test_tag Test can launch pod with measured boot enabled" { @@ -51,6 +52,15 @@ setup() { assert_pod_fail "$pod_config" } +@test "$test_tag Test cannot pull an unencrypted unsigned image from a protected registry" { + setup_signature_files + local container_config="$(new_pod_config "$image_unsigned_protected")" + + echo $container_config + assert_pod_fail "$container_config" + assert_logs_contain "kata" 'Validate image failed: The signatures do not satisfied! Reject reason: \[Match reference failed.\]' +} + @test "$test_tag Test can pull an unencrypted image inside the guest" { create_test_pod @@ -67,15 +77,6 @@ setup() { create_test_pod } -@test "$test_tag Test cannot pull an unencrypted unsigned image from a protected registry" { - setup_signature_files - local container_config="$(new_pod_config "$image_unsigned_protected")" - - echo $container_config - assert_pod_fail "$container_config" - assert_logs_contain 'Validate image failed: The signatures do not satisfied! Reject reason: \[Match reference failed.\]' -} - @test "$test_tag Test can pull an unencrypted unsigned image from an unprotected registry" { setup_signature_files pod_config="$(new_pod_config "$image_unsigned_unprotected")" @@ -90,7 +91,7 @@ setup() { echo $container_config assert_pod_fail "$container_config" - assert_logs_contain 'Validate image failed: The signatures do not satisfied! Reject reason: \[signature verify failed! There is no pubkey can verify the signature!\]' + assert_logs_contain "kata" 'Validate image failed: The signatures do not satisfied! Reject reason: \[signature verify failed! There is no pubkey can verify the signature!\]' } @test "$test_tag Test unencrypted image signed with cosign" { @@ -107,11 +108,16 @@ setup() { echo $container_config assert_pod_fail "$container_config" - assert_logs_contain 'Validate image failed: \[PublicKeyVerifier { key: ECDSA_P256_SHA256_ASN1' + assert_logs_contain "kata" 'Validate image failed: \[PublicKeyVerifier { key: ECDSA_P256_SHA256_ASN1' } - @test "$test_tag Test pull an unencrypted unsigned image from an authenticated registry with correct credentials" { + kubectl delete secret cococred --ignore-not-found + AUTH_USER_NAME=$(echo "$REGISTRY_CREDENTIAL_ENCODED" |base64 -d| cut -d':' -f1) + AUTH_USER_PASSWD=$(echo "$REGISTRY_CREDENTIAL_ENCODED" |base64 -d| cut -d':' -f2) + kubectl create secret docker-registry cococred --docker-server="https://quay.io/kata-containers/confidential-containers-auth" \ + --docker-username="$AUTH_USER_NAME" --docker-password="$AUTH_USER_PASSWD" + if [ "${AA_KBC}" = "offline_fs_kbc" ]; then setup_credentials_files "quay.io/kata-containers/confidential-containers-auth" elif [ "${AA_KBC}" = "cc_kbc" ]; then @@ -126,6 +132,7 @@ setup() { echo $pod_config create_test_pod + kubectl delete secret cococred --ignore-not-found } @test "$test_tag Test cannot pull an image from an authenticated registry with incorrect credentials" { @@ -133,23 +140,46 @@ setup() { skip "As the test requires changing verdictd configuration and restarting its service" fi + kubectl delete secret cococred --ignore-not-found + kubectl create secret docker-registry cococred --docker-server="https://quay.io/kata-containers/confidential-containers-auth" \ + --docker-username="Arandomquaytestaccountthatdoesntexist" --docker-password="password" + REGISTRY_CREDENTIAL_ENCODED="QXJhbmRvbXF1YXl0ZXN0YWNjb3VudHRoYXRkb2VzbnRleGlzdDpwYXNzd29yZAo=" setup_credentials_files "quay.io/kata-containers/confidential-containers-auth" pod_config="$(new_pod_config "${image_authenticated}")" echo "Pod config: ${pod_config}" assert_pod_fail "${pod_config}" - assert_logs_contain 'failed to pull manifest Authentication failure' + assert_logs_contain "containerd" 'failed to fetch oauth token' + kubectl delete secret cococred --ignore-not-found } + @test "$test_tag Test cannot pull an image from an authenticated registry without credentials" { + # TODO - anyway to reset nydus credentials? pod_config="$(new_pod_config "${image_authenticated}")" echo "Pod config: ${pod_config}" assert_pod_fail "${pod_config}" - assert_logs_contain 'failed to pull manifest Not authorized' + + # Print the logs + + sudo journalctl -xe -t kata --since "$test_start_date" -n 100000 + + echo "-- containerd logs:" + sudo journalctl -xe -t containerd --since "$test_start_date" -n 100000 + + echo "-- kubelet logs:" + sudo journalctl -xe -t kubelet --since "$test_start_date" -n 100000 + + assert_logs_contain "containerd" 'failed to resolve reference \\"quay.io/kata-containers/confidential-containers-auth:test\\": pulling from host quay.io failed with status code \[manifests test\]: 401 UNAUTHORIZED' } teardown() { + echo "-- containerd logs:" + sudo journalctl -xe -t containerd --since "$test_start_date" -n 100000 teardown_common + + echo "-- Snapshotter logs:" + sudo journalctl -xe -t snapshotter --since "$test_start_date" -n 100000 } diff --git a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in index 3c8e9d3c1..adea1e483 100644 --- a/integration/kubernetes/confidential/fixtures/pod-config.yaml.in +++ b/integration/kubernetes/confidential/fixtures/pod-config.yaml.in @@ -12,3 +12,5 @@ spec: - name: nginx image: $IMAGE imagePullPolicy: Always + imagePullSecrets: + - name: cococred diff --git a/integration/kubernetes/confidential/lib.sh b/integration/kubernetes/confidential/lib.sh index 9e9d904cc..70fb59772 100755 --- a/integration/kubernetes/confidential/lib.sh +++ b/integration/kubernetes/confidential/lib.sh @@ -121,9 +121,10 @@ assert_pod_fail() { # Note: get the logs since the global $test_start_date. # assert_logs_contain() { - local message="$1" + local log_id="$1" + local message="$2" # Note: with image-rs we get more that the default 1000 lines of logs - journalctl -x -t kata --since "$test_start_date" -n 100000 | grep "$message" + journalctl -x -t $log_id --since "$test_start_date" -n 100000 | grep "$message" } setup_decryption_files_in_guest() { diff --git a/integration/kubernetes/confidential/tests_common.sh b/integration/kubernetes/confidential/tests_common.sh index 177ed450e..943308469 100644 --- a/integration/kubernetes/confidential/tests_common.sh +++ b/integration/kubernetes/confidential/tests_common.sh @@ -31,6 +31,9 @@ setup_common() { configure_cc_containerd "$SAVED_CONTAINERD_CONF_FILE" echo "Reconfigure Kata Containers" + # switch_image_service_offload to on / off is irrelevant + # the the nydus related tests, we'll leave it just because + # it does not cause any harm. switch_image_service_offload on clear_kernel_params add_kernel_params "${original_kernel_params}" @@ -56,6 +59,9 @@ teardown_common() { kubernetes_delete_all_cc_pods_if_any_exists || true clear_kernel_params add_kernel_params "${original_kernel_params}" + # switch_image_service_offload to on / off is irrelevant + # the the nydus related tests, we'll leave it just because + # it does not cause any harm. switch_image_service_offload off disable_full_debug } diff --git a/integration/kubernetes/init.sh b/integration/kubernetes/init.sh index 7f3e72e62..0080f1ae5 100755 --- a/integration/kubernetes/init.sh +++ b/integration/kubernetes/init.sh @@ -24,6 +24,8 @@ CI_JOB=${CI_JOB:-} network_plugin_config="${network_plugin_config:-}" RUNTIME=${RUNTIME:-containerd-shim-kata-v2} RUNTIME_PATH=${RUNTIME_PATH:-$(command -v $RUNTIME)} +IMAGE_OFFLOAD_TO_GUEST="${IMAGE_OFFLOAD_TO_GUEST:-no}" +PULL_ON_HOST="${PULL_ON_HOST:-no}" CRI_RUNTIME="${CRI_RUNTIME:-crio}" untaint_node() { @@ -355,6 +357,17 @@ main() { kubectl create -f "${runtimeclass_files_path}/kata-runtimeclass.yaml" untaint_node + + # Configure snapshotter, if needed. + if [ "${CRI_RUNTIME}" == crio ]; then + return 0 + fi + + if [ "${IMAGE_OFFLOAD_TO_GUEST}" = "yes" ] || [ "${PULL_ON_HOST}" = "yes" ]; then + info "Install nydus-snapshotter" + bash -f "${SCRIPT_PATH}/../../.ci/install_nydus_snapshotter.sh" + bash -f "${SCRIPT_PATH}/../../.ci/containerd_nydus_setup.sh" + fi } main $@