[DEBUG] test sharing image on the host

.ci/ci_job_flags.sh

@@ -40,6 +40,13 @@ init_ci_flags() {
 	# Use the forked version of containerd for Confidential Containers
 	# Valyes: "yes|no"
 	export FORKED_CONTAINERD="no"
+	# Do the pull on the guest using the upstream containerd for Confidential Containers
+	# Values: "yes|no"
+	export IMAGE_OFFLOAD_TO_GUEST="no"
+	# Do the pull on the host using the upstream containerd for Confidential Containers
+	# Values: "yes|no"
+	export PULL_ON_HOST="no"
+	export PULL_ON_HOST_EXPORT_MODE="image_block"
 	# Hypervisor to use
 	export KATA_HYPERVISOR=""
 	# Install k8s
@@ -131,6 +138,50 @@ case "${CI_JOB}" in
 			;;
 	esac
 	;;
+"CC_CRI_CONTAINERD_K8S_IMAGE_OFFLOAD_TO_GUEST")
+	# This job only tests containerd + k8s
+	init_ci_flags
+	export CRI_CONTAINERD="yes"
+	export CRI_RUNTIME="containerd"
+	export KATA_HYPERVISOR="qemu"
+	export KUBERNETES="yes"
+	# Export any CC specific environment variables
+	export KATA_BUILD_CC="yes"
+	export FORKED_CONTAINERD="no"
+	export IMAGE_OFFLOAD_TO_GUEST="yes"
+	export MEASURED_ROOTFS="yes"
+	export AA_KBC="offline_fs_kbc"
+	;;
+"CC_CRI_CONTAINERD_K8S_PULL_ON_HOST_IMAGE_BLOCK")
+	# This job only tests containerd + k8s
+	init_ci_flags
+	export CRI_CONTAINERD="yes"
+	export CRI_RUNTIME="containerd"
+	export KATA_HYPERVISOR="qemu"
+	export KUBERNETES="yes"
+	# Export any CC specific environment variables
+	export KATA_BUILD_CC="yes"
+	export FORKED_CONTAINERD="no"
+	export PULL_ON_HOST="yes"
+	export PULL_ON_HOST_EXPORT_MODE="image_block"
+	export MEASURED_ROOTFS="yes"
+	export AA_KBC="offline_fs_kbc"
+	;;
+"CC_CRI_CONTAINERD_K8S_PULL_ON_HOST_IMAGE_BLOCK_WITH_VERITY")
+	# This job only tests containerd + k8s
+	init_ci_flags
+	export CRI_CONTAINERD="yes"
+	export CRI_RUNTIME="containerd"
+	export KATA_HYPERVISOR="qemu"
+	export KUBERNETES="yes"
+	# Export any CC specific environment variables
+	export KATA_BUILD_CC="yes"
+	export FORKED_CONTAINERD="no"
+	export PULL_ON_HOST="yes"
+	export PULL_ON_HOST_EXPORT_MODE="image_block_with_verity"
+	export MEASURED_ROOTFS="yes"
+	export AA_KBC="offline_fs_kbc"
+	;;
 "CC_SEV_CRI_CONTAINERD_K8S"|"CC_SNP_CRI_CONTAINERD_K8S")
 	init_ci_flags
 	export CRI_CONTAINERD="yes"

.ci/containerd_nydus_setup.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+cidir=$(dirname "$0")
+source "${cidir}/../lib/common.bash"
+
+# Nydus related configurations
+NYDUS_SNAPSHOTTER_BINARY="/usr/local/bin/containerd-nydus-grpc"
+NYDUS_SNAPSHOTTER_TARFS_CONFIG="/usr/local/share/nydus-snapshotter/config-coco-host-sharing.toml"
+NYDUS_SNAPSHOTTER_GUEST_CONFIG="/usr/local/share/nydus-snapshotter/config-coco-guest-pulling.toml"
+NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_CONFIG:-${NYDUS_SNAPSHOTTER_TARFS_CONFIG}}"
+NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE="${PULL_ON_HOST_EXPORT_MODE:-image_block}"
+
+echo "Configure nydus snapshotter"
+if [ "${IMAGE_OFFLOAD_TO_GUEST:-"no"}" == "yes" ]; then
+	echo "Pulling image on the guest"
+	NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_GUEST_CONFIG}"
+else
+	echo "Pulling image on the host | export_mode = ${NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE}"
+	NYDUS_SNAPSHOTTER_CONFIG="${NYDUS_SNAPSHOTTER_TARFS_CONFIG}"
+	sudo sed -i "s/export_mode = .*/export_mode = \"${NYDUS_SNAPSHOTTER_TARFS_EXPORT_MODE}\"/" "$NYDUS_SNAPSHOTTER_CONFIG"
+fi
+
+echo "Start nydus snapshotter"
+systemd-cat -t snapshotter sudo "${NYDUS_SNAPSHOTTER_BINARY}" --config "${NYDUS_SNAPSHOTTER_CONFIG}" --log-to-stdout --log-level debug &
+
+echo "Configure containerd to use the nydus snapshotter"
+
+containerd_config_file="/etc/containerd/config.toml"
+
+snapshotter_socket="/run/containerd-nydus/containerd-nydus-grpc.sock"
+proxy_config="  [proxy_plugins.nydus]\n    type = \"snapshot\"\n    address = \"${snapshotter_socket}\""
+snapshotter_config="      disable_snapshot_annotations = false\n      snapshotter = \"nydus\""
+
+echo -e "[proxy_plugins]" | sudo tee -a "${containerd_config_file}"
+echo -e "${proxy_config}" | sudo tee -a "${containerd_config_file}"
+
+sudo sed -i '/\[plugins.cri.containerd\]/a\'"${snapshotter_config}" "${containerd_config_file}"
+sudo systemctl restart containerd

.ci/install_nydus_snapshotter.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+#
+# Copyright (c) 2023 Intel Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o errtrace
+
+cidir=$(dirname "$0")
+source "${cidir}/lib.sh"
+
+target_dir="/usr/local/"
+nydus_snapshotter_repo=$(get_version "externals.nydus-snapshotter.url")
+nydus_snapshotter_version=$(get_version "externals.nydus-snapshotter.version")
+nydus_snapshotter_repo_dir="${GOPATH}/src/github.com/containerd/nydus-snapshotter"
+nydus_snapshotter_binary_target_dir="${target_dir}/bin"
+nydus_snapshotter_config_target_dir="${target_dir}/share/nydus-snapshotter"
+
+nydus_repo=${nydus_repo:-"https://github.com/dragonflyoss/image-service"}
+nydus_version=${nydus_version:-"v2.3.0-alpha.0"}
+
+arch="$(uname -m)"
+
+clone_nydus_snapshotter_repo() {
+    add_repo_to_git_safe_directory "${nydus_snapshotter_repo_dir}"
+
+    if [ ! -d "${nydus_snapshotter_repo_dir}" ]; then
+        mkdir -p "${nydus_snapshotter_repo_dir}"
+        git clone ${nydus_snapshotter_repo} "${nydus_snapshotter_repo_dir}" || true
+        pushd "${nydus_snapshotter_repo_dir}"
+        git checkout "${nydus_snapshotter_version}"
+        popd
+    fi
+}
+
+build_nydus_snapshotter() {
+    pushd "${nydus_snapshotter_repo_dir}"
+    if [ "${arch}" = "s390x" ]; then
+        export GOARCH=${arch}
+    fi
+    make
+
+    sudo install -D -m 755 "bin/containerd-nydus-grpc" "${nydus_snapshotter_binary_target_dir}/containerd-nydus-grpc"
+    sudo install -D -m 755 "bin/nydus-overlayfs" "${nydus_snapshotter_binary_target_dir}/nydus-overlayfs"
+    rm -rf "${nydus_snapshotter_repo_dir}/bin"
+    popd >/dev/null
+}
+
+download_nydus_snapshotter_config() {
+    tmp_dir=$(mktemp -d -t install-nydus-snapshotter-config-tmp.XXXXXXXXXX)
+    curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/${nydus_snapshotter_version}/misc/snapshotter/config-coco-guest-pulling.toml -o "${tmp_dir}/config-coco-guest-pulling.toml"
+    curl -L https://raw.githubusercontent.com/containerd/nydus-snapshotter/${nydus_snapshotter_version}/misc/snapshotter/config-coco-host-sharing.toml -o "${tmp_dir}/config-coco-host-sharing.toml"
+    sudo install -D -m 644 "${tmp_dir}/config-coco-guest-pulling.toml" "${nydus_snapshotter_config_target_dir}/config-coco-guest-pulling.toml"
+    sudo install -D -m 644 "${tmp_dir}/config-coco-host-sharing.toml" "${nydus_snapshotter_config_target_dir}/config-coco-host-sharing.toml"
+
+}
+
+download_nydus_from_tarball() {
+    if [ "${arch}" = "s390x" ]; then
+        echo "Skip to download nydus for ${arch}, it doesn't work for ${arch} now."
+        return
+    fi
+
+    local goarch="$(${cidir}/kata-arch.sh --golang)"
+    local tarball_url="${nydus_repo}/releases/download/${nydus_version}/nydus-static-${nydus_version}-linux-${goarch}.tgz"
+    echo "Download tarball from ${tarball_url}"
+    tmp_dir=$(mktemp -d -t install-nydus-tmp.XXXXXXXXXX)
+    curl -Ls "$tarball_url" | sudo tar xfz - -C ${tmp_dir} --strip-components=1
+    sudo install -D -m 755 "${tmp_dir}/nydus-image" "${target_dir}/bin/"
+}
+
+download_nydus_from_tarball
+clone_nydus_snapshotter_repo
+build_nydus_snapshotter
+download_nydus_snapshotter_config
+echo "install nydus-snapshotter successful"

.ci/run.sh

@@ -85,7 +85,19 @@ case "${CI_JOB}" in
 		;;
 	"CC_CRI_CONTAINERD_K8S"|"CC_CRI_CONTAINERD_K8S_TDX_QEMU"|"CC_CRI_CONTAINERD_K8S_SE_QEMU"|"CC_CRI_CONTAINERD_K8S_TDX_CLOUD_HYPERVISOR")
 		info "Running Confidential Container tests"
-		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" bash -c "make cc-kubernetes"
+		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="no" PULL_ON_HOST="no" FORKED_CONTAINERD="yes" bash -c "make cc-kubernetes"
+		;;
+	"CC_CRI_CONTAINERD_K8S_IMAGE_OFFLOAD_TO_GUEST")
+		info "Running Confidential Container tests using nydus to offload the image pulling to the guest"
+		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="yes" PULL_ON_HOST="no" bash -c "make cc-kubernetes"
+		;;
+	"CC_CRI_CONTAINERD_K8S_PULL_ON_HOST_IMAGE_BLOCK")
+		info "Running Confidential Container tests using nydus to offload the image pulling to the guest"
+		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="no" PULL_ON_HOST="yes" PULL_ON_HOST_EXPORT_MODE="image_block" FORKED_CONTAINERD="no" bash -c "make cc-kubernetes"
+		;;
+	"CC_CRI_CONTAINERD_K8S_PULL_ON_HOST_IMAGE_BLOCK_WITH_VERITY")
+		info "Running Confidential Container tests using nydus to offload the image pulling to the guest"
+		sudo -E PATH="$PATH" CRI_RUNTIME="containerd" IMAGE_OFFLOAD_TO_GUEST="no" PULL_ON_HOST="yes" PULL_ON_HOST_EXPORT_MODE="image_block_with_verity" FORKED_CONTAINERD="no" bash -c "make cc-kubernetes"
 		;;
 	"CRIO_K8S")
 		echo "INFO: Running kubernetes tests"

integration/confidential/lib.sh

@@ -209,11 +209,14 @@ configure_cc_containerd() {
 	waitForProcess 30 5 "sudo crictl info >/dev/null"
 
 	# Ensure the cc CRI handler is set.
-	local cri_handler=$(sudo crictl info | \
-		jq '.config.containerd.runtimes.kata.cri_handler')
-	if [[ ! "$cri_handler" =~ cc ]]; then
-		sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
-			"$containerd_conf_file"
+
+	if [ "${FORKED_CONTAINERD:-no}" = "yes" ]; then
+		local cri_handler=$(sudo crictl info | \
+			jq '.config.containerd.runtimes.kata.cri_handler')
+		if [[ ! "$cri_handler" =~ cc ]]; then
+			sudo sed -i 's/\([[:blank:]]*\)\(runtime_type = "io.containerd.kata.v2"\)/\1\2\n\1cri_handler = "cc"/' \
+				"$containerd_conf_file"
+		fi
 	fi
 
 	if [ "$(sudo crictl info | jq -r '.config.cni.confDir')" = "null" ]; then

integration/kubernetes/cleanup_env.sh

@@ -40,6 +40,10 @@ main () {
 	info "Stop ${CRI_RUNTIME} service"
 	sudo systemctl stop "${CRI_RUNTIME}"
 
+	sudo kill -9 $(pidof "containerd-nydus-grpc") || true
+	sudo rm -f "/usr/local/bin/nydus-overlayfs"
+	sudo rm -f "/usr/local/bin/nydus-image"
+
 	info "Remove network devices"
 	for dev in cni0 flannel.1; do
 		info "remove device: $dev"

integration/kubernetes/confidential/agent_image.bats

@@ -32,6 +32,7 @@ test_tag="[cc][agent][kubernetes][containerd]"
 
 setup() {
 	setup_common
+	switch_image_service_offload off
 }
 
 @test "$test_tag Test can launch pod with measured boot enabled" {
@@ -51,6 +52,15 @@ setup() {
 	assert_pod_fail "$pod_config"
 }
 
+@test "$test_tag Test cannot pull an unencrypted unsigned image from a protected registry" {
+	setup_signature_files
+	local container_config="$(new_pod_config "$image_unsigned_protected")"
+
+	echo $container_config
+	assert_pod_fail "$container_config"
+	assert_logs_contain "kata" 'Validate image failed: The signatures do not satisfied! Reject reason: \[Match reference failed.\]'
+}
+
 @test "$test_tag Test can pull an unencrypted image inside the guest" {
 	create_test_pod
 
@@ -67,15 +77,6 @@ setup() {
 	create_test_pod
 }
 
-@test "$test_tag Test cannot pull an unencrypted unsigned image from a protected registry" {
-	setup_signature_files
-	local container_config="$(new_pod_config "$image_unsigned_protected")"
-
-	echo $container_config
-	assert_pod_fail "$container_config"
-	assert_logs_contain 'Validate image failed: The signatures do not satisfied! Reject reason: \[Match reference failed.\]'
-}
-
 @test "$test_tag Test can pull an unencrypted unsigned image from an unprotected registry" {
 	setup_signature_files
 	pod_config="$(new_pod_config "$image_unsigned_unprotected")"
@@ -90,7 +91,7 @@ setup() {
 	echo $container_config
 
 	assert_pod_fail "$container_config"
-	assert_logs_contain 'Validate image failed: The signatures do not satisfied! Reject reason: \[signature verify failed! There is no pubkey can verify the signature!\]'
+	assert_logs_contain "kata" 'Validate image failed: The signatures do not satisfied! Reject reason: \[signature verify failed! There is no pubkey can verify the signature!\]'
 }
 
 @test "$test_tag Test unencrypted image signed with cosign" {
@@ -107,11 +108,16 @@ setup() {
 	echo $container_config
 
 	assert_pod_fail "$container_config"
-	assert_logs_contain 'Validate image failed: \[PublicKeyVerifier { key: ECDSA_P256_SHA256_ASN1'
+	assert_logs_contain "kata" 'Validate image failed: \[PublicKeyVerifier { key: ECDSA_P256_SHA256_ASN1'
 }
 
-
 @test "$test_tag Test pull an unencrypted unsigned image from an authenticated registry with correct credentials" {
+	kubectl delete secret cococred --ignore-not-found
+	AUTH_USER_NAME=$(echo "$REGISTRY_CREDENTIAL_ENCODED" |base64 -d| cut -d':' -f1)
+	AUTH_USER_PASSWD=$(echo "$REGISTRY_CREDENTIAL_ENCODED" |base64 -d| cut -d':' -f2)
+	kubectl create secret docker-registry cococred --docker-server="https://quay.io/kata-containers/confidential-containers-auth" \
+	--docker-username="$AUTH_USER_NAME" --docker-password="$AUTH_USER_PASSWD"
+
 	if [ "${AA_KBC}" = "offline_fs_kbc" ]; then
 		setup_credentials_files "quay.io/kata-containers/confidential-containers-auth"
 	elif [ "${AA_KBC}" = "cc_kbc" ]; then
@@ -126,30 +132,54 @@ setup() {
 	echo $pod_config
 
 	create_test_pod
+	kubectl delete secret cococred --ignore-not-found
 }
 
 @test "$test_tag Test cannot pull an image from an authenticated registry with incorrect credentials" {
 	if [ "${AA_KBC}" = "cc_kbc" ]; then
 		skip "As the test requires changing verdictd configuration and restarting its service"
 	fi
 
+	kubectl delete secret cococred --ignore-not-found
+	kubectl create secret docker-registry cococred --docker-server="https://quay.io/kata-containers/confidential-containers-auth" \
+	--docker-username="Arandomquaytestaccountthatdoesntexist" --docker-password="password"
+
 	REGISTRY_CREDENTIAL_ENCODED="QXJhbmRvbXF1YXl0ZXN0YWNjb3VudHRoYXRkb2VzbnRleGlzdDpwYXNzd29yZAo=" setup_credentials_files "quay.io/kata-containers/confidential-containers-auth"
 
 	pod_config="$(new_pod_config "${image_authenticated}")"
 	echo "Pod config: ${pod_config}"
 
 	assert_pod_fail "${pod_config}"
-	assert_logs_contain 'failed to pull manifest Authentication failure'
+	assert_logs_contain "containerd" 'failed to fetch oauth token'
+	kubectl delete secret cococred --ignore-not-found
 }
 
+
 @test "$test_tag Test cannot pull an image from an authenticated registry without credentials" {
+	# TODO - anyway to reset nydus credentials?
 	pod_config="$(new_pod_config "${image_authenticated}")"
 	echo "Pod config: ${pod_config}"
 
 	assert_pod_fail "${pod_config}"
-	assert_logs_contain 'failed to pull manifest Not authorized'
+
+	# Print the logs 
+
+	sudo journalctl -xe -t kata --since "$test_start_date" -n 100000
+
+	echo "-- containerd logs:"
+	sudo journalctl -xe -t containerd --since "$test_start_date" -n 100000
+
+	echo "-- kubelet logs:"
+	sudo journalctl -xe -t kubelet --since "$test_start_date" -n 100000
+
+	assert_logs_contain "containerd" 'failed to resolve reference \\"quay.io/kata-containers/confidential-containers-auth:test\\": pulling from host quay.io failed with status code \[manifests test\]: 401 UNAUTHORIZED'
 }
 
 teardown() {
+	echo "-- containerd logs:"
+	sudo journalctl -xe -t containerd --since "$test_start_date" -n 100000
 	teardown_common
+
+	echo "-- Snapshotter logs:"
+	sudo journalctl -xe -t snapshotter --since "$test_start_date" -n 100000
 }

integration/kubernetes/confidential/fixtures/pod-config.yaml.in

@@ -12,3 +12,5 @@ spec:
   - name: nginx
     image: $IMAGE
     imagePullPolicy: Always
+  imagePullSecrets:
+  - name: cococred

integration/kubernetes/confidential/lib.sh

@@ -121,9 +121,10 @@ assert_pod_fail() {
 # Note: get the logs since the global $test_start_date.
 #
 assert_logs_contain() {
-	local message="$1"
+	local log_id="$1"
+	local message="$2"
 	# Note: with image-rs we get more that the default 1000 lines of logs
-	journalctl -x -t kata --since "$test_start_date" -n 100000 | grep "$message"
+	journalctl -x -t $log_id --since "$test_start_date" -n 100000 | grep "$message"
 }
 
 setup_decryption_files_in_guest() {