From 851009c6cc3d3f3909ac13c0d77a132b4c1034c4 Mon Sep 17 00:00:00 2001 From: krishjainx Date: Sat, 8 Jul 2023 00:43:58 +0200 Subject: [PATCH 01/40] Install docker as a system extension, use manglefs script ( convert the unit dependencies from wants to upholds); one sysext for all docker for now --- build_library/build_image_util.sh | 26 +++++-------------- manglefs_docker | 11 ++++++++ .../coreos-base/coreos/coreos-0.0.1.ebuild | 1 - 3 files changed, 17 insertions(+), 21 deletions(-) create mode 100755 manglefs_docker diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 26348b0a79e..5086a8659e5 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -626,26 +626,12 @@ finish_image() { local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" - # Copy in packages from the torcx store that are marked as being on disk - if [ -n "${FLAGS_torcx_manifest}" ]; then - for pkg in $(torcx_manifest::get_pkg_names "${FLAGS_torcx_manifest}"); do - local default_version="$(torcx_manifest::default_version "${FLAGS_torcx_manifest}" "${pkg}")" - for version in $(torcx_manifest::get_versions "${FLAGS_torcx_manifest}" "${pkg}"); do - local on_disk_path="$(torcx_manifest::local_store_path "${FLAGS_torcx_manifest}" "${pkg}" "${version}")" - if [[ -n "${on_disk_path}" ]]; then - local casDigest="$(torcx_manifest::get_digest "${FLAGS_torcx_manifest}" "${pkg}" "${version}")" - sudo cp "${FLAGS_torcx_root}/pkgs/${BOARD}/${pkg}/${casDigest}/${pkg}:${version}.torcx.tgz" \ - "${root_fs_dir}${on_disk_path}" - sudo tar xf "${root_fs_dir}${on_disk_path}" -C "${root_fs_dir}" --wildcards "./usr/share/SLSA" - if [[ "${version}" == "${default_version}" ]]; then - # Create the default symlink for this package - sudo ln -fns "${on_disk_path##*/}" \ - "${root_fs_dir}/${on_disk_path%/*}/${pkg}:com.coreos.cl.torcx.tgz" - fi - fi - done - done - fi +# Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker + emerge-"${FLAGS_board}" app-containers/docker + sudo "$(dirname ${BASH_SOURCE[0]})/../build_sysext" --board="${BOARD}" --build_dir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="$(dirname ${BASH_SOURCE[0]})/../manglefs_docker" docker-flatcar app-containers/docker + sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ + sudo mkdir -p "${root_fs_dir}"/etc/extensions/· + sudo ln -sf /usr/share/flatcar/docker-flatcar.raw "${root_fs_dir}"/etc/extensions/docker-flatcar.raw # Only enable rootfs verification on prod builds. local disable_read_write="${FLAGS_FALSE}" diff --git a/manglefs_docker b/manglefs_docker new file mode 100755 index 00000000000..c13bdcd93e2 --- /dev/null +++ b/manglefs_docker @@ -0,0 +1,11 @@ +#!/bin/bash + +set -euo pipefail +rootfs="${1}" + +# docker: +mkdir -p "${rootfs}/usr/lib/systemd/system/sockets.target.d" +{ echo "[Unit]"; echo "Upholds=docker.socket"; } > "${rootfs}/usr/lib/systemd/system/sockets.target.d/10-docker-socket.conf" +# containerd: +mkdir -p "${rootfs}/usr/lib/systemd/system/multi-user.target.d" +{ echo "[Unit]"; echo "Upholds=containerd.service"; } > "${rootfs}/usr/lib/systemd/system/multi-user.target.d/10-containerd-service.conf" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild index 5313de0531c..67d4fc6e02c 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild @@ -95,7 +95,6 @@ RDEPEND="${RDEPEND} app-arch/xz-utils app-arch/zstd app-arch/tar - app-arch/torcx app-arch/unzip app-arch/zip app-arch/ncompress From c026d9ed538f5ff0da867223c89db245f20363a8 Mon Sep 17 00:00:00 2001 From: krishjainx Date: Sat, 8 Jul 2023 00:44:45 +0200 Subject: [PATCH 02/40] Remove app-arch/torcx from coreos-overlay --- .../app-arch/torcx/files/compat-wrapper.sh | 33 ----------- .../app-arch/torcx/files/docker-1.12-no.json | 11 ---- .../app-arch/torcx/files/dockerd-wrapper.sh | 41 ------------- .../app-arch/torcx/files/torcx.target | 7 --- .../app-arch/torcx/files/vendor.json | 11 ---- .../app-arch/torcx/metadata.xml | 4 -- .../app-arch/torcx/torcx-0.2.0-r9.ebuild | 1 - .../app-arch/torcx/torcx-9999.ebuild | 58 ------------------- 8 files changed, 166 deletions(-) delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/compat-wrapper.sh delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/docker-1.12-no.json delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/dockerd-wrapper.sh delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/torcx.target delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/vendor.json delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-arch/torcx/metadata.xml delete mode 120000 sdk_container/src/third_party/coreos-overlay/app-arch/torcx/torcx-0.2.0-r9.ebuild delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-arch/torcx/torcx-9999.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/compat-wrapper.sh b/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/compat-wrapper.sh deleted file mode 100644 index 122162decae..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/compat-wrapper.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -set -e - -bin=${0##*/} -seal=/run/metadata/torcx - -if [ -z "${bin}" ] -then - echo 'Failed to determine the executed program name.' 1>&2 - exit 1 -fi - -if [ -s "${seal}" ] -then - . "${seal}" -else - echo "The program ${bin} is managed by torcx, which did not run." 1>&2 - exit 1 -fi - -if [ -z "${TORCX_BINDIR-}" ] -then - echo "The torcx seal file ${seal} is invalid." 1>&2 - exit 1 -fi - -if [ ! -x "${TORCX_BINDIR}/${bin}" ] -then - echo "The current torcx profile did not install a ${bin} program." 1>&2 - exit 1 -fi - -PATH="${TORCX_BINDIR}${PATH:+:${PATH}}" exec "${TORCX_BINDIR}/${bin}" "$@" diff --git a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/docker-1.12-no.json b/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/docker-1.12-no.json deleted file mode 100644 index 4480c4e7670..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/docker-1.12-no.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "kind": "profile-manifest-v0", - "value": { - "images": [ - { - "name": "docker", - "reference": "com.coreos.cl" - } - ] - } -} diff --git a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/dockerd-wrapper.sh b/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/dockerd-wrapper.sh deleted file mode 100644 index 19c440bddab..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/dockerd-wrapper.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash -# Wrapper for launching docker daemons with selinux default on -# This wrapper script has been deprecated (euank: 2017-05-09) and is retained -# for backwards compatibility. - -set -e - -parse_docker_args() { - local flag - while [[ $# -gt 0 ]]; do - flag="$1" - shift - - # treat --flag=foo and --flag foo identically - if [[ "${flag}" == *=* ]]; then - set -- "${flag#*=}" "$@" - flag="${flag%=*}" - fi - - case "${flag}" in - --selinux-enabled) - ARG_SELINUX="$1" - shift - ;; - *) - # ignore everything else - ;; - esac - done -} - -parse_docker_args "$@" - -USE_SELINUX="" -# Do not override selinux if it is already explicitly configured. -if [[ -z "${ARG_SELINUX}" ]]; then - # If unspecified, default off - USE_SELINUX="--selinux-enabled=false" -fi - -exec dockerd "$@" ${USE_SELINUX} diff --git a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/torcx.target b/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/torcx.target deleted file mode 100644 index adfd836204e..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/torcx.target +++ /dev/null @@ -1,7 +0,0 @@ -[Unit] -Description=Verify torcx succeeded -DefaultDependencies=no -AssertPathExists=/run/metadata/torcx - -[Install] -WantedBy=basic.target diff --git a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/vendor.json b/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/vendor.json deleted file mode 100644 index 4480c4e7670..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/files/vendor.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "kind": "profile-manifest-v0", - "value": { - "images": [ - { - "name": "docker", - "reference": "com.coreos.cl" - } - ] - } -} diff --git a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/metadata.xml b/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/metadata.xml deleted file mode 100644 index 097975e3adc..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/metadata.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/torcx-0.2.0-r9.ebuild b/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/torcx-0.2.0-r9.ebuild deleted file mode 120000 index a82f2db3523..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/torcx-0.2.0-r9.ebuild +++ /dev/null @@ -1 +0,0 @@ -torcx-9999.ebuild \ No newline at end of file diff --git a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/torcx-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/torcx-9999.ebuild deleted file mode 100644 index a7b94f7b224..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-arch/torcx/torcx-9999.ebuild +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2017-2018 CoreOS, Inc.. All rights reserved. -# Distributed under the terms of the GNU General Public License v2 - -EAPI=7 -CROS_WORKON_PROJECT="flatcar/torcx" -CROS_WORKON_LOCALNAME="torcx" -CROS_WORKON_REPO="https://github.com" -COREOS_GO_PACKAGE="github.com/flatcar/torcx" -COREOS_GO_GO111MODULE="off" - -if [[ "${PV}" == 9999 ]]; then - KEYWORDS="~amd64 ~arm64" -else - CROS_WORKON_COMMIT="212a152e3eaf50232762011a5d9d29a4f0f8dfb9" # flatcar-master - KEYWORDS="amd64 arm64" -fi - -inherit coreos-go cros-workon systemd - -DESCRIPTION="torcx is a boot-time addon manager for immutable systems" -HOMEPAGE="https://github.com/flatcar/torcx" -LICENSE="Apache-2.0" -SLOT="0" - -src_compile() { - CGO_ENABLED=0 go_export - ${EGO} build -v \ - -p "$(makeopts_jobs)" \ - -ldflags "-X ${COREOS_GO_PACKAGE}/pkg/version.VERSION=${PV}" \ - -o "bin/${ARCH}/torcx" \ - -tags containers_image_openpgp \ - "${COREOS_GO_PACKAGE}" -} - -src_install() { - local generatordir=/usr/lib/systemd/system-generators - local vendordir=/usr/share/torcx - local libcoreosdir=/usr/lib/flatcar - - # Install generator and userland. - exeinto "${generatordir}" - newexe "${S}/bin/${ARCH}/torcx" torcx-generator - dosym ../systemd/system-generators/torcx-generator "${libcoreosdir}/torcx" - systemd_dounit "${FILESDIR}/torcx.target" - - insinto "${vendordir}/profiles" - doins "${FILESDIR}/docker-1.12-no.json" - doins "${FILESDIR}/vendor.json" - dodir "${vendordir}/store" - - # Preserve program paths for torcx packages. - newbin "${FILESDIR}/compat-wrapper.sh" docker - for link in {docker-,}{containerd{,-shim},runc} ctr docker-{init,proxy} dockerd tini - do ln -fns docker "${ED}/usr/bin/${link}" - done - exeinto /usr/lib/flatcar - newexe "${FILESDIR}/dockerd-wrapper.sh" dockerd -} From 3d17f65088165803cc8e90fe7b53b6073ed1b7b5 Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Sun, 16 Jul 2023 01:26:33 +0530 Subject: [PATCH 03/40] Create seperate docker and containerd sysext --- build_library/build_image_util.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 5086a8659e5..cde00efe843 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -626,12 +626,21 @@ finish_image() { local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" -# Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker + # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker + echo "app-containers/containerd-1.6.16" >> ~/trunk/src/third_party/coreos-overlay/profiles/coreos/base/package.provided emerge-"${FLAGS_board}" app-containers/docker - sudo "$(dirname ${BASH_SOURCE[0]})/../build_sysext" --board="${BOARD}" --build_dir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="$(dirname ${BASH_SOURCE[0]})/../manglefs_docker" docker-flatcar app-containers/docker + sudo "$(dirname ${BASH_SOURCE[0]})/../build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="$(dirname ${BASH_SOURCE[0]})/../manglefs_docker" docker-flatcar app-containers/docker sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ - sudo mkdir -p "${root_fs_dir}"/etc/extensions/· + sudo mkdir -p "${root_fs_dir}"/etc/extensions/ sudo ln -sf /usr/share/flatcar/docker-flatcar.raw "${root_fs_dir}"/etc/extensions/docker-flatcar.raw + sed -i '/containerd/d' ~/trunk/src/third_party/coreos-overlay/profiles/coreos/base/package.provided + + + emerge-"${FLAGS_board}" app-containers/containerd + sudo "$(dirname ${BASH_SOURCE[0]})/../build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" containerd-flatcar app-containers/containerd + sudo install -m 0644 -D "${BUILD_DIR}/containerd-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ + sudo mkdir -p "${root_fs_dir}"/etc/extensions/ + sudo ln -sf /usr/share/flatcar/containerd-flatcar.raw "${root_fs_dir}"/etc/extensions/containerd-flatcar.raw # Only enable rootfs verification on prod builds. local disable_read_write="${FLAGS_FALSE}" From 2be2e884b3a24fb9ffb2cf91a3c64544b866876d Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Mon, 17 Jul 2023 18:27:03 +0530 Subject: [PATCH 04/40] Add manglefs script for containerd --- build_library/build_image_util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index cde00efe843..720d505813b 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -637,7 +637,7 @@ finish_image() { emerge-"${FLAGS_board}" app-containers/containerd - sudo "$(dirname ${BASH_SOURCE[0]})/../build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" containerd-flatcar app-containers/containerd + sudo "$(dirname ${BASH_SOURCE[0]})/../build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="$(dirname ${BASH_SOURCE[0]})/../manglefs_containerd" containerd-flatcar app-containers/containerd sudo install -m 0644 -D "${BUILD_DIR}/containerd-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ sudo mkdir -p "${root_fs_dir}"/etc/extensions/ sudo ln -sf /usr/share/flatcar/containerd-flatcar.raw "${root_fs_dir}"/etc/extensions/containerd-flatcar.raw From e5f1ecbae3bb6ce6833fe745418e581ed3d1985a Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Mon, 17 Jul 2023 18:27:44 +0530 Subject: [PATCH 05/40] Only keep docker stuff in manglefs_docker --- manglefs_docker | 4 ---- 1 file changed, 4 deletions(-) diff --git a/manglefs_docker b/manglefs_docker index c13bdcd93e2..4cb69961036 100755 --- a/manglefs_docker +++ b/manglefs_docker @@ -3,9 +3,5 @@ set -euo pipefail rootfs="${1}" -# docker: mkdir -p "${rootfs}/usr/lib/systemd/system/sockets.target.d" { echo "[Unit]"; echo "Upholds=docker.socket"; } > "${rootfs}/usr/lib/systemd/system/sockets.target.d/10-docker-socket.conf" -# containerd: -mkdir -p "${rootfs}/usr/lib/systemd/system/multi-user.target.d" -{ echo "[Unit]"; echo "Upholds=containerd.service"; } > "${rootfs}/usr/lib/systemd/system/multi-user.target.d/10-containerd-service.conf" From e6bb5470c4c5aa8334b66a5df8136f64899ff711 Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Mon, 17 Jul 2023 18:30:55 +0530 Subject: [PATCH 06/40] Add mangle script for containerd --- manglefs_containerd | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 manglefs_containerd diff --git a/manglefs_containerd b/manglefs_containerd new file mode 100755 index 00000000000..ff5c5d36952 --- /dev/null +++ b/manglefs_containerd @@ -0,0 +1,7 @@ +#!/bin/bash + +set -euo pipefail +rootfs="${1}" + +mkdir -p "${rootfs}/usr/lib/systemd/system/multi-user.target.d" +{ echo "[Unit]"; echo "Upholds=containerd.service"; } > "${rootfs}/usr/lib/systemd/system/multi-user.target.d/10-containerd-service.conf" From 9709a64239bccb130f64642184efd6750a198e44 Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Tue, 18 Jul 2023 16:06:20 +0530 Subject: [PATCH 07/40] incorporate suggestions from Jeremi --- build_library/build_image_util.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 720d505813b..78864ffac7d 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -627,17 +627,18 @@ finish_image() { local disk_img="${BUILD_DIR}/${image_name}" # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker - echo "app-containers/containerd-1.6.16" >> ~/trunk/src/third_party/coreos-overlay/profiles/coreos/base/package.provided + mkdir -p ${PORTAGE_CONFIGROOT}/etc/portage/profile + echo $(equery -q --no-color list --format '$cpv' -po app-containers/containerd | grep -v 9999 | head -n 1) > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed emerge-"${FLAGS_board}" app-containers/docker - sudo "$(dirname ${BASH_SOURCE[0]})/../build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="$(dirname ${BASH_SOURCE[0]})/../manglefs_docker" docker-flatcar app-containers/docker + sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" docker-flatcar app-containers/docker sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ sudo mkdir -p "${root_fs_dir}"/etc/extensions/ sudo ln -sf /usr/share/flatcar/docker-flatcar.raw "${root_fs_dir}"/etc/extensions/docker-flatcar.raw - sed -i '/containerd/d' ~/trunk/src/third_party/coreos-overlay/profiles/coreos/base/package.provided + rm ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided emerge-"${FLAGS_board}" app-containers/containerd - sudo "$(dirname ${BASH_SOURCE[0]})/../build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="$(dirname ${BASH_SOURCE[0]})/../manglefs_containerd" containerd-flatcar app-containers/containerd + sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script=""${SCRIPTS_DIR}/manglefs_containerd" containerd-flatcar app-containers/containerd sudo install -m 0644 -D "${BUILD_DIR}/containerd-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ sudo mkdir -p "${root_fs_dir}"/etc/extensions/ sudo ln -sf /usr/share/flatcar/containerd-flatcar.raw "${root_fs_dir}"/etc/extensions/containerd-flatcar.raw From 4b00bc14a4f6f8553a667071934173ea5ebdae40 Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Tue, 18 Jul 2023 16:12:42 +0530 Subject: [PATCH 08/40] tail, not head --- build_library/build_image_util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 78864ffac7d..55956015ead 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -628,7 +628,7 @@ finish_image() { # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker mkdir -p ${PORTAGE_CONFIGROOT}/etc/portage/profile - echo $(equery -q --no-color list --format '$cpv' -po app-containers/containerd | grep -v 9999 | head -n 1) > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed + echo $(equery -q --no-color list --format '$cpv' -po app-containers/containerd | grep -v 9999 | tail -n 1) > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed emerge-"${FLAGS_board}" app-containers/docker sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" docker-flatcar app-containers/docker sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ From abe8bf683a29058af7ef3a3a5a56235bb443964f Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Wed, 19 Jul 2023 01:10:46 +0530 Subject: [PATCH 09/40] Use helper function query_available_package() --- build_library/build_image_util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 55956015ead..ae20ca1bf82 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -628,7 +628,7 @@ finish_image() { # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker mkdir -p ${PORTAGE_CONFIGROOT}/etc/portage/profile - echo $(equery -q --no-color list --format '$cpv' -po app-containers/containerd | grep -v 9999 | tail -n 1) > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed + query_available_package containerd > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed emerge-"${FLAGS_board}" app-containers/docker sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" docker-flatcar app-containers/docker sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ From 4e8b27631940c0852ee64fe2834eda1a24515c78 Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Thu, 20 Jul 2023 17:34:55 +0530 Subject: [PATCH 10/40] Remove duplicate making directory root_fs_dir/etc/extensions --- build_library/build_image_util.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index ae20ca1bf82..f030b23cdd1 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -640,7 +640,6 @@ finish_image() { emerge-"${FLAGS_board}" app-containers/containerd sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script=""${SCRIPTS_DIR}/manglefs_containerd" containerd-flatcar app-containers/containerd sudo install -m 0644 -D "${BUILD_DIR}/containerd-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ - sudo mkdir -p "${root_fs_dir}"/etc/extensions/ sudo ln -sf /usr/share/flatcar/containerd-flatcar.raw "${root_fs_dir}"/etc/extensions/containerd-flatcar.raw # Only enable rootfs verification on prod builds. From fe949f59005ad1fcf8e7d79319c5378e9e6cce38 Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Thu, 20 Jul 2023 18:02:21 +0530 Subject: [PATCH 11/40] Make sure binary packages are built in build_packages already --- build_library/build_image_util.sh | 3 --- .../coreos-devel/board-packages/board-packages-0.0.1.ebuild | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index f030b23cdd1..f1f5e41a85f 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -629,15 +629,12 @@ finish_image() { # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker mkdir -p ${PORTAGE_CONFIGROOT}/etc/portage/profile query_available_package containerd > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed - emerge-"${FLAGS_board}" app-containers/docker sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" docker-flatcar app-containers/docker sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ sudo mkdir -p "${root_fs_dir}"/etc/extensions/ sudo ln -sf /usr/share/flatcar/docker-flatcar.raw "${root_fs_dir}"/etc/extensions/docker-flatcar.raw rm ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided - - emerge-"${FLAGS_board}" app-containers/containerd sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script=""${SCRIPTS_DIR}/manglefs_containerd" containerd-flatcar app-containers/containerd sudo install -m 0644 -D "${BUILD_DIR}/containerd-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ sudo ln -sf /usr/share/flatcar/containerd-flatcar.raw "${root_fs_dir}"/etc/extensions/containerd-flatcar.raw diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild index e9a082a32ee..d7369534956 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild @@ -29,6 +29,8 @@ RDEPEND=" sys-boot/grub sys-firmware/edk2-ovmf-bin ) + app-containers/containerd + app-containers/docker app-emulation/amazon-ssm-agent app-emulation/wa-linux-agent coreos-base/coreos From f6f5c32417a57fd8e30654db49ae2cd2b95f1482 Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Fri, 21 Jul 2023 18:49:12 +0530 Subject: [PATCH 12/40] add information about the packages contained in the built sysext --- build_library/build_image_util.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index f1f5e41a85f..16cf1e358ef 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -260,6 +260,18 @@ image_packages_portage() { ROOT="$1" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ equery --no-color list --format '$cpv::$repo' '*' } + +# List dependencies for a package runtime dependencies + +function package_run_dependencies() ( + pkg=${1:?} + ebuild=$(equery-${BOARD} w "${pkg}") + function inherit() { : ; } + . "${ebuild}" + echo ${RDEPEND} +) + + # List packages implicitly contained in rootfs, such as in torcx packages or # initramfs. image_packages_implicit() { @@ -295,6 +307,12 @@ image_packages_implicit() { [ -z "${FLAGS_torcx_manifest}" ] || torcx_manifest::sources_on_disk "${FLAGS_torcx_manifest}" | while read pkg ; do query_available_package "${pkg}" ; done + + + # Include source packages of all sysext images installed on disk. + for docker_containerd_package in $(package_run_dependencies docker) $(package_run_dependencies containerd); do + query_available_package ${docker_containerd_package} + done } # Generate a list of packages installed in an image. From 4633cc3e7c37f80234b5f68af060cb45d718c46a Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Mon, 24 Jul 2023 18:43:51 +0530 Subject: [PATCH 13/40] Fix syntax errors, quoting --- build_library/build_image_util.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 16cf1e358ef..80f19b8c0cb 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -311,7 +311,7 @@ image_packages_implicit() { # Include source packages of all sysext images installed on disk. for docker_containerd_package in $(package_run_dependencies docker) $(package_run_dependencies containerd); do - query_available_package ${docker_containerd_package} + query_available_package "${docker_containerd_package}" ; done } @@ -645,7 +645,7 @@ finish_image() { local disk_img="${BUILD_DIR}/${image_name}" # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker - mkdir -p ${PORTAGE_CONFIGROOT}/etc/portage/profile + mkdir -p "${PORTAGE_CONFIGROOT}"/etc/portage/profile query_available_package containerd > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" docker-flatcar app-containers/docker sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ @@ -653,7 +653,7 @@ finish_image() { sudo ln -sf /usr/share/flatcar/docker-flatcar.raw "${root_fs_dir}"/etc/extensions/docker-flatcar.raw rm ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided - sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script=""${SCRIPTS_DIR}/manglefs_containerd" containerd-flatcar app-containers/containerd + sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_containerd" containerd-flatcar app-containers/containerd sudo install -m 0644 -D "${BUILD_DIR}/containerd-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ sudo ln -sf /usr/share/flatcar/containerd-flatcar.raw "${root_fs_dir}"/etc/extensions/containerd-flatcar.raw From 298e28394450979c97180d74e66390dcc5e011df Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Tue, 25 Jul 2023 11:32:34 +0530 Subject: [PATCH 14/40] fix permissions issue --- build_library/build_image_util.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 80f19b8c0cb..ddfe376c832 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -646,12 +646,12 @@ finish_image() { # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker mkdir -p "${PORTAGE_CONFIGROOT}"/etc/portage/profile - query_available_package containerd > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed + sudo query_available_package containerd > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" docker-flatcar app-containers/docker sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ sudo mkdir -p "${root_fs_dir}"/etc/extensions/ sudo ln -sf /usr/share/flatcar/docker-flatcar.raw "${root_fs_dir}"/etc/extensions/docker-flatcar.raw - rm ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided + sudo rm ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_containerd" containerd-flatcar app-containers/containerd sudo install -m 0644 -D "${BUILD_DIR}/containerd-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ From 3c4639b0ce3177986b2aebabcee68a2695a72fed Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Tue, 25 Jul 2023 13:47:44 +0530 Subject: [PATCH 15/40] use sudo_clobber function --- build_library/build_image_util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index ddfe376c832..8a978890233 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -646,7 +646,7 @@ finish_image() { # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker mkdir -p "${PORTAGE_CONFIGROOT}"/etc/portage/profile - sudo query_available_package containerd > ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed + query_available_package containerd | sudo_clobber ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" docker-flatcar app-containers/docker sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ sudo mkdir -p "${root_fs_dir}"/etc/extensions/ From cc029bb5ca31c7394f09438db3affdf105d8f20f Mon Sep 17 00:00:00 2001 From: Krish Jain Date: Wed, 26 Jul 2023 00:37:09 +0530 Subject: [PATCH 16/40] use DEVCONTAINER=1 --- build_library/dev_container_util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index a0c07477cd1..dd172987fb9 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -110,7 +110,7 @@ create_dev_container() { # The remount services are provided by coreos-base/coreos-init systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service" - finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}" + DEVCONTAINER=1 finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}" declare -a files_to_evaluate declare -a compressed_images From 9cede02f86217f36afa2a25650b312b0fd8d2ae0 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 5 Oct 2023 15:24:42 +0200 Subject: [PATCH 17/40] dev_container_util.sh: remove spurious DEVCONTAINER=1 Signed-off-by: Thilo Fromm --- build_library/dev_container_util.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index dd172987fb9..a0c07477cd1 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -110,7 +110,7 @@ create_dev_container() { # The remount services are provided by coreos-base/coreos-init systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service" - DEVCONTAINER=1 finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}" + finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}" declare -a files_to_evaluate declare -a compressed_images From 9837c35ec485c8e8e196688f38e6bd99688a9986 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 5 Oct 2023 15:25:50 +0200 Subject: [PATCH 18/40] build_sysext: add sysext dependency handling This change adds 2 optional command line parameters to build_sysext to handle dependencies in stacked sysexts. The command line parameters allow exporting portage package db information into a separate squashfs image as well as using package db information exported by a previous sysext build. --generate_pkginfo will generate a separate squashfs _pkginfo.raw which contains the sysext's /var/db/pkgs. --base_pkginfo=[:[:...]] will use the supplied paths to pkginfo squashfses as additional lowerdirs when creating a sysext. Useage example: ./build_sysext --generate_pkginfo containerd app-containers/containerd ./build_sysext --base_pkginfo=containerd_pkginfo.raw docker app-containers/docker will create a containerd.raw sysext and a dependent docker.raw sysext that does not have the containerd dependency installed. Both sysexts must be merged together in order for docker to work. Signed-off-by: Thilo Fromm --- build_sysext | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/build_sysext b/build_sysext index c783f57fa57..bc7b0646e5d 100755 --- a/build_sysext +++ b/build_sysext @@ -27,6 +27,10 @@ DEFINE_string image_builddir '' \ "Custom directory to build the sysext in. Defaults to a 'sysext' sub-directory of the directory the squashfs base image resides in; '${default_imagedir}/sysext' by default." DEFINE_string manglefs_script '' \ "A path to executable that will customize the rootfs of the sysext image." +DEFINE_boolean generate_pkginfo "${FLAGS_FALSE}" \ + "Generate an additional squashfs '_pkginfo.raw' with portage package meta-information (/var/db ...). Useful for creating sysext dependencies; see 'base_pkginfo' below." +DEFINE_string base_pkginfo "" \ + "Colon-separated list of pkginfo squashfs paths / files generated via 'generate_pkginfo' to base this sysext on. The corresponding base sysexts are expected to be merged with the sysext generated." DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \ "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS." @@ -126,6 +130,10 @@ cleanup() { ) umount "${dirs[@]}" 2>/dev/null || true rm -rf "${dirs[@]}" || true + if [[ -d "${BUILD_DIR}/img-pkginfo" ]] ; then + umount "${BUILD_DIR}/img-pkginfo"/* 2>/dev/null || true + rm -rf "${BUILD_DIR}/img-pkginfo" || true + fi } # Set up trap to execute cleanup() on script exit @@ -134,11 +142,33 @@ trap cleanup EXIT ARCH=$(_get_sysext_arch "${FLAGS_board}") cleanup +# If we need to handle pkginfo squashfs files, create mount points under +# ${BUILD_DIR}/img-pkginfo, mount the squashfs images, and add the mount paths to +# the list of lowerdirs. +pkginfo_lowerdirs="" +if [[ -n "${FLAGS_base_pkginfo}" ]] ; then + for entry in $(echo ${FLAGS_base_pkginfo} | sed 's/:/ /g'); do + ppath="$(readlink -f "${entry}")" + if [[ ! -f "${ppath}" ]] ; then + error "--base_pkginfo contains invalid entries." + error "Pkginfo file '${ppath}' does not exist." + die "Full --base_pkginfo: '${FLAGS_base_pkginfo}'" + fi + + pfile="$(basename "${ppath}")" + pmdir="${BUILD_DIR}/img-pkginfo/${pfile}" + mkdir -p "${pmdir}" + mount -rt squashfs -o loop,nodev "${ppath}" "${pmdir}" + pkginfo_lowerdirs="${pkginfo_lowerdirs}:${pmdir}" + info "Added packageinfo from '${ppath}' to base layers." + done +fi + mkdir "${BUILD_DIR}/fs-root" mount -rt squashfs -o loop,nodev "${FLAGS_squashfs_base}" "${BUILD_DIR}/fs-root" mkdir "${BUILD_DIR}/install-root" mkdir "${BUILD_DIR}/workdir" -mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root" +mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root${pkginfo_lowerdirs}",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root" VERSION_BOARD=$(grep "^VERSION=" ${BUILD_DIR}/fs-root/usr/lib/os-release | cut -d = -f 2-) if [ "$VERSION_BOARD" != "$FLATCAR_VERSION" ]; then warn "Base squashfs version: $VERSION_BOARD" @@ -186,6 +216,13 @@ if [[ -n "${FLAGS_manglefs_script}" ]]; then "${FLAGS_manglefs_script}" "${BUILD_DIR}/install-root" fi +if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then + info " Creating pkginfo squashfs '${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw'" + mkdir -p "${BUILD_DIR}/img-pkginfo/var/db" + cp -R "${BUILD_DIR}/install-root/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/" + mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend +fi + info "Removing non-/usr directories from sysext image" for entry in "${BUILD_DIR}/install-root"/*; do if [[ "${entry}" = */usr ]]; then From aae3fd826083940b35ab7280f7e6fa22468cd67d Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 5 Oct 2023 15:36:53 +0200 Subject: [PATCH 19/40] prod_image_util.sh: remove pkgdb tarball (replaced by sysext) Signed-off-by: Thilo Fromm --- build_library/prod_image_util.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 8ae6255b1ff..9aa86eae7e3 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -77,7 +77,6 @@ create_prod_image() { local image_initrd_contents="${image_name%.bin}_initrd_contents.txt" local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt" local image_disk_usage="${image_name%.bin}_disk_usage.txt" - local image_pkgdb="${image_name%.bin}_pkgdb.tar.xz" local image_sysext_base="${image_name%.bin}_sysext.squashfs" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" @@ -102,7 +101,6 @@ create_prod_image() { || die_notrace "coreos-au-key is missing the 'official' use flag" fi - tar -cf "${BUILD_DIR}/${image_pkgdb}" -C "${root_fs_dir}" var/cache/edb var/db/pkg sudo cp -a "${root_fs_dir}" "${BUILD_DIR}/root_fs_dir2" sudo rsync -a --delete "${BUILD_DIR}/configroot/etc/portage" "${BUILD_DIR}/root_fs_dir2/etc" sudo mksquashfs "${BUILD_DIR}/root_fs_dir2" "${BUILD_DIR}/${image_sysext_base}" -noappend From 8f8f262f194adb32226fa93a81bf1774ec6ad019 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 5 Oct 2023 19:11:42 +0200 Subject: [PATCH 20/40] torcx: remove from scripts, use docker+containerd sysexts This change removes torcx libraries, references, and commandline options from build automation scripts and from build_library/. Containerd and docker are shipped via sysexts which are included in the base image. Signed-off-by: Thilo Fromm --- build_image | 10 -- build_library/build_image_util.sh | 48 ++--- build_library/release_util.sh | 43 ----- build_library/torcx_manifest.sh | 150 ---------------- build_packages | 14 -- build_torcx_store | 287 ------------------------------ core_dev_sign_update | 15 -- 7 files changed, 27 insertions(+), 540 deletions(-) delete mode 100644 build_library/torcx_manifest.sh delete mode 100755 build_torcx_store diff --git a/build_image b/build_image index 6a2377d69f0..c7bef33f10c 100755 --- a/build_image +++ b/build_image @@ -33,10 +33,6 @@ DEFINE_string base_pkg "coreos-base/coreos" \ "The base portage package to base the build off of (only applies to prod images)" DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \ "The base portage package to base the build off of (only applies to dev containers)" -DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \ - "The torcx manifest describing torcx packages for this image (or blank for none)" -DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \ - "Directory in which torcx packages can be found. Will update the default --torcx_manifest if set." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string disk_layout "" \ @@ -91,11 +87,6 @@ switch_to_strict_mode check_gsutil_opts -# Patch around default values not being able to depend on other flags. -if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then - FLAGS_torcx_manifest="${FLAGS_torcx_root}/${FLAGS_board}/latest/torcx_manifest.json" -fi - # If downloading packages is enabled ensure the board is configured properly. if [[ ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then "${SRC_ROOT}/scripts/setup_board" --board="${FLAGS_board}" \ @@ -110,7 +101,6 @@ fi . "${BUILD_LIBRARY_DIR}/prod_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/dev_container_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 -. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1 PROD_IMAGE=0 diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 8a978890233..f2ba149a647 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -272,8 +272,7 @@ function package_run_dependencies() ( ) -# List packages implicitly contained in rootfs, such as in torcx packages or -# initramfs. +# List packages implicitly contained in rootfs, such as in initramfs. image_packages_implicit() { local profile="${BUILD_DIR}/configroot/etc/portage/profile" @@ -303,12 +302,6 @@ image_packages_implicit() { done < "${profile}/package.provided" fi - # Include source packages of all torcx images installed on disk. - [ -z "${FLAGS_torcx_manifest}" ] || - torcx_manifest::sources_on_disk "${FLAGS_torcx_manifest}" | - while read pkg ; do query_available_package "${pkg}" ; done - - # Include source packages of all sysext images installed on disk. for docker_containerd_package in $(package_run_dependencies docker) $(package_run_dependencies containerd); do query_available_package "${docker_containerd_package}" ; @@ -535,8 +528,6 @@ EOF # Add /usr/share/SLSA reports for packages indirectly contained within the rootfs # If the package is available in BOARD_ROOT accesses it from there, otherwise # needs to download binpkg. -# Reports for torcx packages are also included when adding the torcx package to -# rootfs. insert_extra_slsa() { info "Inserting additional SLSA file" local rootfs="$1" @@ -644,18 +635,33 @@ finish_image() { local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" - # Ship the docker systemd-sysext image and rip out torcx in same go; TODO: create seperate sysext images for containerd and docker - mkdir -p "${PORTAGE_CONFIGROOT}"/etc/portage/profile - query_available_package containerd | sudo_clobber ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided # use a temporary package.provided to make emerge believe the dependencies are already installed - sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" docker-flatcar app-containers/docker - sudo install -m 0644 -D "${BUILD_DIR}/docker-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ + info "Creating containerd and docker sysexts." + + sudo "${SCRIPTS_DIR}/build_sysext" \ + --board="${BOARD}" \ + --image_builddir="${BUILD_DIR}" \ + --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ + --manglefs_script="${SCRIPTS_DIR}/manglefs_containerd" \ + --generate_pkginfo \ + containerd app-containers/containerd + + sudo "${SCRIPTS_DIR}/build_sysext" \ + --board="${BOARD}" \ + --image_builddir=${BUILD_DIR} \ + --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ + --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" \ + --base_pkginfo="${BUILD_DIR}/containerd_pkginfo.raw" \ + docker app-containers/docker + + sudo mkdir -p "${root_fs_dir}"/usr/share/flatcar/sysext + sudo install -m 0644 -D "${BUILD_DIR}/containerd.raw" "${root_fs_dir}"/usr/share/flatcar/sysext/ + sudo install -m 0644 -D "${BUILD_DIR}/docker.raw" "${root_fs_dir}"/usr/share/flatcar/sysext/ + + # Install symlinks into /etc/extensions - this will be picked up by the logic to populate + # /usr/share/flatcar/etc below, so it will end up below /usr in the final image. sudo mkdir -p "${root_fs_dir}"/etc/extensions/ - sudo ln -sf /usr/share/flatcar/docker-flatcar.raw "${root_fs_dir}"/etc/extensions/docker-flatcar.raw - sudo rm ${PORTAGE_CONFIGROOT}/etc/portage/profile/package.provided - - sudo "${SCRIPTS_DIR}/build_sysext" --board="${BOARD}" --image_builddir=${BUILD_DIR} --squashfs_base="${BUILD_DIR}/${image_sysext_base}" --manglefs_script="${SCRIPTS_DIR}/manglefs_containerd" containerd-flatcar app-containers/containerd - sudo install -m 0644 -D "${BUILD_DIR}/containerd-flatcar.raw" "${root_fs_dir}"/usr/share/flatcar/ - sudo ln -sf /usr/share/flatcar/containerd-flatcar.raw "${root_fs_dir}"/etc/extensions/containerd-flatcar.raw + sudo ln -sf /usr/share/flatcar/sysext/containerd.raw "${root_fs_dir}"/etc/extensions/containerd.raw + sudo ln -sf /usr/share/flatcar/sysext/docker.raw "${root_fs_dir}"/etc/extensions/docker.raw # Only enable rootfs verification on prod builds. local disable_read_write="${FLAGS_FALSE}" diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 16d33f2c933..21d5aa58057 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -5,7 +5,6 @@ GSUTIL_OPTS= UPLOAD_ROOT= UPLOAD_PATH= -TORCX_UPLOAD_ROOT= UPLOAD_DEFAULT=${FLAGS_FALSE} DEFAULT_IMAGE_COMPRESSION_FORMAT="bz2" @@ -13,7 +12,6 @@ DEFAULT_IMAGE_COMPRESSION_FORMAT="bz2" _user="${USER}" [[ ${USER} == "root" ]] && _user="${SUDO_USER}" : ${FLATCAR_UPLOAD_ROOT:=gs://users.developer.core-os.net/${_user}} -: ${FLATCAR_TORCX_UPLOAD_ROOT:=${FLATCAR_UPLOAD_ROOT}/torcx} unset _user DEFINE_boolean parallel ${FLAGS_TRUE} \ @@ -30,12 +28,6 @@ DEFINE_string download_root "" \ "HTTP download prefix, board/version/etc will be appended." DEFINE_string download_path "" \ "HTTP download path, overrides --download_root." -DEFINE_string torcx_upload_root "${FLATCAR_TORCX_UPLOAD_ROOT}" \ - "Tectonic torcx package and manifest Upload prefix. Must be a gs:// URL." -DEFINE_string tectonic_torcx_download_root "" \ - "HTTP download prefix for tectonic torcx packages and manifests." -DEFINE_string tectonic_torcx_download_path "" \ - "HTTP download path, overrides --tectonic_torcx_download_root." DEFINE_string sign "" \ "Sign all files to be uploaded with the given GPG key." DEFINE_string sign_digests "" \ @@ -164,15 +156,6 @@ check_gsutil_opts() { UPLOAD_ROOT="${FLAGS_upload_root%%/}" fi - if [[ -n "${FLAGS_torcx_upload_root}" ]]; then - if [[ "${FLAGS_torcx_upload_root}" != gs://* ]] \ - && [[ "${FLAGS_torcx_upload_root}" != rsync://* ]] ; then - die_notrace "--torcx_upload_root must be a gs:// or rsync:// URL" - fi - # Make sure the path doesn't end with a slash - TORCX_UPLOAD_ROOT="${FLAGS_torcx_upload_root%%/}" - fi - if [[ -n "${FLAGS_upload_path}" ]]; then if [[ "${FLAGS_upload_path}" != gs://* ]] \ && [[ "${FLAGS_upload_path}" != rsync://* ]] ; then @@ -373,29 +356,3 @@ download_image_url() { echo "${download_path}/$1" } - -# Translate the configured torcx upload URL to a download url -# This is similar to the download_image_url, other than assuming the release -# bucket is the tectonic_torcx one. -download_tectonic_torcx_url() { - if [[ ${FLAGS_upload} -ne ${FLAGS_TRUE} ]]; then - echo "$1" - return 0 - fi - - local download_root="${FLAGS_tectonic_torcx_download_root:-${TORCX_UPLOAD_ROOT}}" - - local download_path - if [[ -n "${FLAGS_tectonic_torcx_download_path}" ]]; then - download_path="${FLAGS_tectonic_torcx_download_path%%/}" - else - download_path="${download_root%%/}" - fi - - # Just in case download_root was set from UPLOAD_ROOT - if [[ "${download_path}" == gs://* ]]; then - download_path="http://${download_path#gs://}" - fi - - echo "${download_path}/$1" -} diff --git a/build_library/torcx_manifest.sh b/build_library/torcx_manifest.sh deleted file mode 100644 index c44402279c2..00000000000 --- a/build_library/torcx_manifest.sh +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) 2017 The Container Linux by CoreOS Authors. All rights -# reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# torcx_manifest.sh contains helper functions for creating, editing, and -# reading torcx manifest files. - -# create_empty creates an empty torcx manfiest at the given path. -function torcx_manifest::create_empty() { - local path="${1}" - jq '.' > "${path}" < "${path}" -} - -# get_pkg_names returns the list of packages in a given manifest. Each package -# may have one or more versions associated with it. -# -# Example: -# pkg_name_arr=($(torcx_manifest::get_pkg_names "torcx_manifest.json")) -function torcx_manifest::get_pkg_names() { - local file="${1}" - jq -r '.value.packages[].name' < "${file}" -} - -# local_store_path returns the in-container-linux store path a given package + -# version combination should exist at. It returns the empty string if the -# package shouldn't exist on disk. -function torcx_manifest::local_store_path() { - local file="${1}" - local name="${2}" - local version="${3}" - jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .locations[] | select(.path).path" < "${file}" -} - -# get_digest returns the cas digest for a given package version -function torcx_manifest::get_digest() { - local file="${1}" - local name="${2}" - local version="${3}" - jq -r ".value.packages[] | select(.name == \"${name}\") | .versions[] | select(.version == \"${version}\") | .casDigest" < "${file}" -} - -# get_digests returns the list of digests for a given package. -function torcx_manifest::get_digests() { - local file="${1}" - local name="${2}" - jq -r ".value.packages[] | select(.name == \"${name}\").versions[].casDigest" < "${file}" -} - -# get_versions returns the list of versions for a given package. -function torcx_manifest::get_versions() { - local file="${1}" - local name="${2}" - jq -r ".value.packages[] | select(.name == \"${name}\").versions[].version" < "${file}" -} - -# default_version returns the default version for a given package, or an empty string if there isn't one. -function torcx_manifest::default_version() { - local file="${1}" - local name="${2}" - jq -r ".value.packages[] | select(.name == \"${name}\").defaultVersion" < "${file}" -} - -# sources_on_disk returns the list of source packages of all torcx images installed on disk -function torcx_manifest::sources_on_disk() { - local file="${1}" - local torcx_pkg="" - jq -r ".value.packages[].versions[] | select(.locations[].path).metaPackage" < "${file}" | - while read torcx_pkg; do - torcx_dependencies "${torcx_pkg}" | tr ' ' '\n' - done -} - -# Print the first level of runtime dependencies for a torcx meta-package. -function torcx_dependencies() ( - pkg=${1:?} - ebuild=$(equery-${BOARD} w "${pkg}") - function inherit() { : ; } - . "${ebuild}" - echo ${RDEPEND} -) diff --git a/build_packages b/build_packages index 5989e45e3f9..e2b343b50d7 100755 --- a/build_packages +++ b/build_packages @@ -36,12 +36,6 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ "Don't update toolchain automatically." DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \ "Don't run the chroot upgrade automatically; use with care." -DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \ - "Directory in which to place torcx stores and manifests (named by board/version)" -DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \ - "Don't build a new torcx store from the updated sysroot." -DEFINE_string torcx_extra_pkg_url "" \ - "URL to directory where the torcx packages will be available for downloading" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -329,13 +323,5 @@ test_image_content "${BOARD_ROOT}" # upload packages if enabled upload_packages -# Build a new torcx store with the updated packages, passing flags through. -if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then - "${SCRIPTS_DIR}"/build_torcx_store \ - --board="${BOARD}" \ - --output_root="${FLAGS_torcx_output_root}" \ - --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}" -fi - info "Builds complete" command_completed diff --git a/build_torcx_store b/build_torcx_store deleted file mode 100755 index 7e4018c0d58..00000000000 --- a/build_torcx_store +++ /dev/null @@ -1,287 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2017 The CoreOS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -. "$(dirname "$0")/common.sh" || exit 1 - -# Script must run inside the chroot -assert_inside_chroot - -assert_not_root_user - -# Developer-visible flags. -DEFINE_string board "${DEFAULT_BOARD}" \ - "The board to build packages for." -DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \ - "Directory in which to place torcx stores and manifests (named by board/version)" -DEFINE_string extra_pkg_url "" \ - "URL to directory where the torcx packages will be available for downloading" - -# include upload options -. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 - -FLAGS_HELP="usage: $(basename $0) [flags] [images] - -This script builds a collection of torcx images to be installed into a torcx -store. By default, all supported images are built, but a list of images can be -given as command arguments. Note that their order matters, since the version -specified last will get the default reference symlink. -" -show_help_if_requested "$@" - -# The following options are advanced options, only available to those willing -# to read the source code. They are not shown in help output, since they are -# not needed for the typical developer workflow. -DEFINE_integer build_attempt 1 \ - "The build attempt for this image build." -DEFINE_string group developer \ - "The update group." -DEFINE_string version '' \ - "Overrides version number in name to this version." - -# Parse command line -FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV}" - -# Only now can we die on error. shflags functions leak non-zero error codes, -# so will die prematurely if 'switch_to_strict_mode' is specified before now. -switch_to_strict_mode - -# Initialize upload options -check_gsutil_opts - -# Define BUILD_DIR and set_build_symlinks. -. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 -. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 -. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 -. "${BUILD_LIBRARY_DIR}/torcx_manifest.sh" || exit 1 - -TORCX_CAS_ROOT="${FLAGS_output_root}/pkgs/${BOARD}" - -# Build and install a package configured as part of a torcx image. -function torcx_build() ( - tmproot=${1:?} - shift - pkgs=( "${@}" ) - - export LDFLAGS=-Wl,-rpath,/ORIGIN/../lib - export PKGDIR="${tmproot}/var/lib/portage/pkgs" - - # Allow the meta-package to install bashrc to customize the builds. - [ -s "${tmproot}/etc/portage/bashrc" ] && - . "${tmproot}/etc/portage/bashrc" - - # Build binary packages using dev files in the board root. - emerge-${BOARD} \ - --jobs="${NUM_JOBS}" \ - --buildpkg \ - --buildpkgonly \ - --nodeps \ - --oneshot \ - --verbose \ - --root-deps=rdeps \ - "${pkgs[@]}" - - # Install the binary packages in the temporary torcx image root. - emerge-${BOARD} \ - --jobs="${NUM_JOBS}" \ - --nodeps \ - --oneshot \ - --verbose \ - --root="${tmproot}" \ - --root-deps=rdeps \ - --sysroot="${tmproot}" \ - --usepkgonly \ - "${pkgs[@]}" -) - -# Create a torcx image from the given meta-package. -function torcx_package() { - local pkg="app-torcx/${1##*/}" - local name=${pkg%-[0-9]*} - local version=${pkg:${#name}+1} - local manifest_path="${2}" - local type="${3}" - local extra_pkg_url="${4}" - local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile - local pkg_cas_file pkg_cas_root - local pkg_locations=() - local name=${name##*/} - local version=${version%%-r*} - - # Run in a subshell to clean tmproot and tmppkgroot up without - # clobbering this shell's EXIT trap. - ( - # Set up the base package layout to dump everything into /bin and /lib. - # tmproot is what the packages are installed into. - # A subset of the files from tmproot are then moved into tmppkgroot, - # which is then archived and uploaded. - tmproot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) - tmppkgroot=$(sudo mktemp --tmpdir="${BUILD_DIR}" -d) - trap "sudo rm -rf '${tmproot}' '${tmppkgroot}'" EXIT - sudo chmod 0755 "${tmproot}" "${tmppkgroot}" - sudo mkdir -p "${tmproot}"/{.torcx,bin,lib,usr} - sudo ln -fns ../bin "${tmproot}/usr/bin" - sudo ln -fns ../lib "${tmproot}/usr/lib" - sudo ln -fns lib "${tmproot}/usr/lib64" - sudo ln -fns bin "${tmproot}/usr/sbin" - sudo ln -fns lib "${tmproot}/lib64" - sudo ln -fns bin "${tmproot}/sbin" - - # Install the meta-package and its direct dependencies. - torcx_build "${tmproot}" "=${pkg}" $(torcx_dependencies "${pkg}") - - # by convention, the first dependency in a torcx package is the primary - # source package - rdepends=($(torcx_dependencies "${pkg}")) - source_pkg="${rdepends[0]#=}" - - # Pluck out shared libraries and SONAME links. - sudo mv "${tmproot}"/{lib,tmplib} - sudo rm -fr "${tmproot}/tmplib/debug" - sudo find "${tmproot}/tmplib" -name 'lib*.so' -type l -delete - sudo mkdir -p "${tmproot}/lib" - sudo find "${tmproot}/tmplib" -name 'lib*.so*' \ - -exec mv -t "${tmproot}/lib/" {} + - - # Rewrite any units for transparent activation from the torcx root. - if [ -e "${tmproot}/tmplib/systemd/system" ] - then - sudo mkdir -p "${tmproot}/lib/systemd" - sudo mv "${tmproot}/tmplib/systemd/system" \ - "${tmproot}/lib/systemd/" - sudo find "${tmproot}/lib/systemd/system" -type f -exec sed -i \ - -e '/^\[Unit]/aRequires=torcx.target\nAfter=torcx.target' \ - -e '/^\[Service]/aEnvironmentFile=/run/metadata/torcx' \ - -e "/^\[Service]/aEnvironment=TORCX_IMAGEDIR=/${name}" \ - -e 's,/usr/s\?bin/,${TORCX_BINDIR}/,g' \ - -e 's,^\([^ ]*=\)\(.{TORCX_BINDIR}\)/,\1/usr/bin/env PATH=\2:${PATH} \2/,' {} + - fi - - # Network configuration can be installed unmodified. - if [ -e "${tmproot}/tmplib/systemd/network" ] - then - sudo mkdir -p "${tmproot}/lib/systemd" - sudo mv "${tmproot}/tmplib/systemd/network" \ - "${tmproot}/lib/systemd/" - fi - - # Rewrite RPATHs to use the real $ORIGIN value. - find -H "${tmproot}"/{bin,lib} -type f | - while read file - do - ( - rpath=$(sudo patchelf --print-rpath "${file}" 2>/dev/null) && - test "${rpath#/ORIGIN/}" != "${rpath}" && - sudo patchelf --set-rpath "${rpath/#?/\$}" "${file}" - ) || : # Set $? to 0 or the pipeline fails and -e quits. - done - - # Move anything we plan to package to its root. - sudo mv "${tmproot}"/{.torcx,bin,lib} "${tmppkgroot}" - if [ -e "${tmproot}/usr/share" ] - then - sudo mkdir "${tmppkgroot}/usr" - sudo mv "${tmproot}/usr/share" "${tmppkgroot}/usr/" - fi - - tmpfile="${BUILD_DIR}/${name}:${version}.torcx.tgz" - tar --force-local --selinux --xattrs -C "${tmppkgroot}" -czf "${tmpfile}" . - sha512sum=$(sha512sum "${tmpfile}" | awk '{print $1}') - - # TODO(euank): this opaque digest, if it were reproducible, could save - # users from having to download things that haven't changed. - # For now, use the sha512sum of the final image. - # Ideally we should move to something more like a casync digest or tarsum. - # The reason this is currently not being done is because to do that we - # *MUST* ensure that a given pair of (digest, sha512sum) referenced in - # a previous torcx package remains correct. - # Because this code, as written, clobbers existing things with the same - # digest (but the sha512sum of the .torcx.tgz can differ, e.g. due to ctime) - # that property doesn't hold. - # To switch this back to a reprodicble digest, we *must* never clobber - # existing objects (and thus re-use their sha512sum here). - digest="${sha512sum}" - - pkg_cas_root="${TORCX_CAS_ROOT}/${name}/${digest}" - pkg_cas_file="${pkg_cas_root}/${name}:${version}.torcx.tgz" - mkdir -p "${pkg_cas_root}" - mv "${tmpfile}" "${pkg_cas_file}" - - update_default=false - if [[ "${type}" == "default" ]]; then - update_default=true - pkg_locations+=("/usr/share/torcx/store/${name}:${version}.torcx.tgz") - fi - if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then - pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")") - fi - if [[ -n "${extra_pkg_url}" ]]; then - pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz") - fi - torcx_manifest::add_pkg "${manifest_path}" \ - "${name}" \ - "${version}" \ - "sha512-${sha512sum}" \ - "${digest}" \ - "${source_pkg}" \ - "${pkg}" \ - "${update_default}" \ - "${pkg_locations[@]}" - ) -} - -# This list defines every torcx image that goes into the vendor store for the -# current branch's release version. Note that the default reference symlink -# for each package will point at the last version specified. This can handle -# swapping default package versions for different OS releases by reordering. -DEFAULT_IMAGES=( - =app-torcx/docker-20.10 -) - -# This list contains extra images which will be uploaded and included in the -# generated manifest, but won't be included in the vendor store. -EXTRA_IMAGES=( -) - -mkdir -p "${BUILD_DIR}" -manifest_path="${BUILD_DIR}/torcx_manifest.json" -torcx_manifest::create_empty "${manifest_path}" -for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do - torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}" -done -for pkg in "${EXTRA_IMAGES[@]}"; do - torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}" -done - -set_build_symlinks latest "${FLAGS_group}-latest" - -# Upload the pkgs referenced by this manifest -for pkg in $(torcx_manifest::get_pkg_names "${manifest_path}"); do - for digest in $(torcx_manifest::get_digests "${manifest_path}" "${pkg}"); do - # no need to sign; the manifest includes their shasum and is signed. - upload_files \ - 'torcx pkg' \ - "${TORCX_UPLOAD_ROOT}/pkgs/${BOARD}/${pkg}/${digest}" \ - "" \ - "${TORCX_CAS_ROOT}/${pkg}/${digest}"/*.torcx.tgz - done -done - -# Upload the manifest -# Note: the manifest is uploaded to 'UPLOAD_ROOT' rather than -# 'TORCX_UPLOAD_ROOT'. -# For non-release builds, those two locations will be the same, so it usually -# won't matter. -# However, for release builds, torcx packages may be uploaded directly to their -# final location, while the manifest still has to go through build bucket in -# order to get signed. -sign_and_upload_files \ - 'torcx manifest' \ - "${UPLOAD_ROOT}/torcx/manifests/${BOARD}/${FLATCAR_VERSION}" \ - "" \ - "${manifest_path}" - -# vim: tabstop=8 softtabstop=4 shiftwidth=8 expandtab diff --git a/core_dev_sign_update b/core_dev_sign_update index 46704653a4b..6ed848ee34c 100755 --- a/core_dev_sign_update +++ b/core_dev_sign_update @@ -69,18 +69,3 @@ else --armor --detach-sign "${data_dir}/update.hash" fi echo "=== Update payload signed successfully. ===" - -echo "=== Verifying torcx manifest... ===" -gpg2 --verify "${data_dir}/torcx_manifest.json.sig" -echo "=== Signing torcx manifest... ===" -if [[ -z "${FLAGS_gpg_key}" ]]; then - gpg2 \ - --output "${output_dir}/torcx_manifest.json.sig.$(whoami)" \ - --detach-sign --armor "${data_dir}/torcx_manifest.json" -else - gpg2 \ - --local-user "$FLAGS_gpg_key" \ - --output "${output_dir}/torcx_manifest.json.sig.$(whoami)" \ - --detach-sign --armor "${data_dir}/torcx_manifest.json" -fi -echo "=== Torcx manifest signed successfully. ===" From 8e01a2c8f9f782a22686eecfed8916f16de7a4c5 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 6 Oct 2023 10:14:18 +0200 Subject: [PATCH 21/40] build_image: add sysext command line option This change refactors sysext builds during build_image and generalises the code (no hard-coded containerd and docker anymore). A command line option is added to build_image for sysexts to include in the OS image. It defaults to containerd and docker but may be set to arbitrary packages. The command line supports simple depenencies, i.e. the "docker" sysext will re-use package information from the "containerd" sysext and not include another containerd. Signed-off-by: Thilo Fromm --- build_image | 4 +- build_library/build_image_util.sh | 28 -------- build_library/prod_image_util.sh | 67 +++++++++++++++++++ .../sysext_mangle_app-containers_containerd | 0 .../sysext_mangle_app-containers_docker | 0 5 files changed, 70 insertions(+), 29 deletions(-) rename manglefs_containerd => build_library/sysext_mangle_app-containers_containerd (100%) rename manglefs_docker => build_library/sysext_mangle_app-containers_docker (100%) diff --git a/build_image b/build_image index c7bef33f10c..a10a410f1d7 100755 --- a/build_image +++ b/build_image @@ -33,6 +33,8 @@ DEFINE_string base_pkg "coreos-base/coreos" \ "The base portage package to base the build off of (only applies to prod images)" DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \ "The base portage package to base the build off of (only applies to dev containers)" +DEFINE_string base_sysexts "app-containers/containerd,app-containers/docker" \ + "Comma-separated list of packages to build into sysexts and include with OS image and update payload. Must be in order of dependencies, base sysexts come first." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string disk_layout "" \ @@ -165,7 +167,7 @@ fi if [[ "${PROD_IMAGE}" -eq 1 ]]; then IMAGE_BUILD_TYPE="prod" - create_prod_image ${FLATCAR_PRODUCTION_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} ${FLAGS_base_pkg} + create_prod_image ${FLATCAR_PRODUCTION_IMAGE_NAME} ${DISK_LAYOUT} ${FLAGS_group} ${FLAGS_base_pkg} ${FLAGS_base_sysexts} if [[ ${FLAGS_generate_update} -eq ${FLAGS_TRUE} ]]; then generate_update "${FLATCAR_PRODUCTION_IMAGE_NAME}" ${DISK_LAYOUT} elif [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]; then diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index f2ba149a647..7073da75d30 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -635,34 +635,6 @@ finish_image() { local install_grub=0 local disk_img="${BUILD_DIR}/${image_name}" - info "Creating containerd and docker sysexts." - - sudo "${SCRIPTS_DIR}/build_sysext" \ - --board="${BOARD}" \ - --image_builddir="${BUILD_DIR}" \ - --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ - --manglefs_script="${SCRIPTS_DIR}/manglefs_containerd" \ - --generate_pkginfo \ - containerd app-containers/containerd - - sudo "${SCRIPTS_DIR}/build_sysext" \ - --board="${BOARD}" \ - --image_builddir=${BUILD_DIR} \ - --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ - --manglefs_script="${SCRIPTS_DIR}/manglefs_docker" \ - --base_pkginfo="${BUILD_DIR}/containerd_pkginfo.raw" \ - docker app-containers/docker - - sudo mkdir -p "${root_fs_dir}"/usr/share/flatcar/sysext - sudo install -m 0644 -D "${BUILD_DIR}/containerd.raw" "${root_fs_dir}"/usr/share/flatcar/sysext/ - sudo install -m 0644 -D "${BUILD_DIR}/docker.raw" "${root_fs_dir}"/usr/share/flatcar/sysext/ - - # Install symlinks into /etc/extensions - this will be picked up by the logic to populate - # /usr/share/flatcar/etc below, so it will end up below /usr in the final image. - sudo mkdir -p "${root_fs_dir}"/etc/extensions/ - sudo ln -sf /usr/share/flatcar/sysext/containerd.raw "${root_fs_dir}"/etc/extensions/containerd.raw - sudo ln -sf /usr/share/flatcar/sysext/docker.raw "${root_fs_dir}"/etc/extensions/docker.raw - # Only enable rootfs verification on prod builds. local disable_read_write="${FLAGS_FALSE}" if [[ "${IMAGE_BUILD_TYPE}" == "prod" ]]; then diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 9aa86eae7e3..6259f0dbdd2 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -52,6 +52,58 @@ extract_prod_gcc() { package_provided "${gcc}" } +# Create a sysext from a package and install it to the OS image. +# Conventions: +# - For each /, __pkginfo will be built. Can be used in subsequent calls +# to build dependent sysexts. +# - If ${BUILD_LIBRARY_DIR}/sysext_mangle__ exists it will be used as FS mangle script +# when building the sysext. +# +create_prod_sysext() { + local install_root="$1" + local base_image="$2" + local grp_pkg="$3" + local pkginfo="${4:-}" + + local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package + local pkginfo_opt="" + local manglefs_opt="" + + local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw" + + # Include previous sysexts' pkginfo if supplied + if [[ -n "${pkginfo}" ]] ; then + if [[ ! -f "${BUILD_DIR}/${pkginfo}" ]] ; then + die "Sysext build '${grp_pkg}': unable to find package info at '${BUILD_DIR}/${pkginfo}'." + fi + msg="${msg} w/ package info '${pkginfo}'" + pkginfo_opt="--base_pkginfo=${BUILD_DIR}/${pkginfo}" + fi + + # Include FS mangle script if present + if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then + manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" + msg="${msg}, FS mangle script 'sysext_mangle_${name}'" + fi + + info "${msg}." + + sudo "${SCRIPTS_DIR}/build_sysext" \ + --board="${BOARD}" \ + --image_builddir="${BUILD_DIR}" \ + --squashfs_base="${base_image}" \ + --generate_pkginfo \ + ${manglefs_opt} ${pkginfo_opt} \ + "${name}" "${grp_pkg}" + + sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext + sudo install -m 0644 -D "${BUILD_DIR}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/ + + sudo mkdir -p "${install_root}"/etc/extensions/ + sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw" +} +# -- + create_prod_image() { local image_name="$1" local disk_layout="$2" @@ -62,6 +114,8 @@ create_prod_image() { exit 1 fi + local base_sysexts="$5" + info "Building production image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" local image_contents="${image_name%.bin}_contents.txt" @@ -136,6 +190,19 @@ EOF # Remove source locale data, only need to ship the compiled archive. sudo rm -rf ${root_fs_dir}/usr/share/i18n/ + if [[ -n "${base_sysexts}" ]] ; then + local grp_pkg="" + local prev_pkginfo="" + for grp_pkg in ${base_sysexts//,/ }; do + create_prod_sysext "${root_fs_dir}"\ + "${BUILD_DIR}/${image_sysext_base}" \ + "${grp_pkg}" \ + "${prev_pkginfo}" + prev_pkginfo="${grp_pkg//\//_}_pkginfo.raw" + done + fi + + # Finish image will move files from /etc to /usr/share/flatcar/etc. finish_image \ "${image_name}" \ "${disk_layout}" \ diff --git a/manglefs_containerd b/build_library/sysext_mangle_app-containers_containerd similarity index 100% rename from manglefs_containerd rename to build_library/sysext_mangle_app-containers_containerd diff --git a/manglefs_docker b/build_library/sysext_mangle_app-containers_docker similarity index 100% rename from manglefs_docker rename to build_library/sysext_mangle_app-containers_docker From c589fb8e562e55da3ae1451c1cb690d4d9e13579 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 6 Oct 2023 11:41:56 +0200 Subject: [PATCH 22/40] coreos-overlay: removed all remaining app-torcx artifacts Signed-off-by: Thilo Fromm --- .../app-torcx/docker/docker-20.10.ebuild | 32 ------------------- .../docker/files/docker-17.03-manifest.json | 29 ----------------- .../docker/files/docker-19.03-manifest.json | 29 ----------------- .../docker/files/docker-20.10-manifest.json | 30 ----------------- .../coreos-overlay/profiles/categories | 1 - .../bootengine/bootengine-9999.ebuild | 1 - 6 files changed, 122 deletions(-) delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-torcx/docker/docker-20.10.ebuild delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-17.03-manifest.json delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-19.03-manifest.json delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-20.10-manifest.json diff --git a/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/docker-20.10.ebuild b/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/docker-20.10.ebuild deleted file mode 100644 index 54a268670b4..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/docker-20.10.ebuild +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2017-2018 CoreOS, Inc.. All rights reserved. -# Distributed under the terms of the GNU General Public License v2 - -EAPI=7 - -DESCRIPTION="Packages to be installed in a torcx image for Docker" - -LICENSE="GPL-2" -SLOT="0" -KEYWORDS="amd64 arm64" - -# Explicitly list all packages that will be built into the image. -RDEPEND=" - ~app-containers/docker-20.10.24 - ~app-containers/docker-cli-20.10.24 - ~app-containers/containerd-1.7.7 - ~app-containers/docker-proxy-0.8.0_p20230118 - ~app-containers/runc-1.1.9 - ~dev-libs/libltdl-2.4.7 - ~sys-process/tini-0.19.0 -" - -S="${WORKDIR}" - -src_install() { - insinto /.torcx - newins "${FILESDIR}/${P}-manifest.json" manifest.json - - # Enable the Docker socket by default. - local unitdir=/usr/lib/systemd/system - dosym ../docker.socket "${unitdir}/sockets.target.wants/docker.socket" -} diff --git a/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-17.03-manifest.json b/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-17.03-manifest.json deleted file mode 100644 index 190f9e8538c..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-17.03-manifest.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "kind": "image-manifest-v0", - "value": { - "bin": [ - "/bin/containerd", - "/bin/containerd-shim", - "/bin/ctr", - "/bin/docker", - "/bin/docker-containerd", - "/bin/docker-containerd-shim", - "/bin/docker-init", - "/bin/docker-proxy", - "/bin/docker-runc", - "/bin/dockerd", - "/bin/runc", - "/bin/tini" - ], - "network": [ - "/lib/systemd/network/50-docker.network", - "/lib/systemd/network/90-docker-veth.network" - ], - "units": [ - "/lib/systemd/system/containerd.service", - "/lib/systemd/system/docker.service", - "/lib/systemd/system/docker.socket", - "/lib/systemd/system/sockets.target.wants" - ] - } -} diff --git a/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-19.03-manifest.json b/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-19.03-manifest.json deleted file mode 100644 index 190f9e8538c..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-19.03-manifest.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "kind": "image-manifest-v0", - "value": { - "bin": [ - "/bin/containerd", - "/bin/containerd-shim", - "/bin/ctr", - "/bin/docker", - "/bin/docker-containerd", - "/bin/docker-containerd-shim", - "/bin/docker-init", - "/bin/docker-proxy", - "/bin/docker-runc", - "/bin/dockerd", - "/bin/runc", - "/bin/tini" - ], - "network": [ - "/lib/systemd/network/50-docker.network", - "/lib/systemd/network/90-docker-veth.network" - ], - "units": [ - "/lib/systemd/system/containerd.service", - "/lib/systemd/system/docker.service", - "/lib/systemd/system/docker.socket", - "/lib/systemd/system/sockets.target.wants" - ] - } -} diff --git a/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-20.10-manifest.json b/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-20.10-manifest.json deleted file mode 100644 index 418acbdcd9d..00000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-torcx/docker/files/docker-20.10-manifest.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "kind": "image-manifest-v0", - "value": { - "bin": [ - "/bin/containerd", - "/bin/containerd-shim", - "/bin/ctr", - "/bin/docker", - "/bin/docker-containerd", - "/bin/docker-containerd-shim", - "/bin/docker-init", - "/bin/docker-proxy", - "/bin/docker-runc", - "/bin/dockerd", - "/bin/runc", - "/bin/tini" - ], - "network": [ - "/lib/systemd/network/50-docker.network", - "/lib/systemd/network/90-docker-veth.network" - ], - "units": [ - "/lib/systemd/system/containerd.service", - "/lib/systemd/system/docker.service", - "/lib/systemd/system/docker.socket", - "/lib/systemd/system/sockets.target.wants", - "/lib/systemd/system/multi-user.target.wants" - ] - } -} diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/categories b/sdk_container/src/third_party/coreos-overlay/profiles/categories index 8afb9b905b5..2d661545c21 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/categories +++ b/sdk_container/src/third_party/coreos-overlay/profiles/categories @@ -1,3 +1,2 @@ -app-torcx coreos-base coreos-devel diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild index 637cb31b838..fed305a3e39 100644 --- a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild @@ -37,7 +37,6 @@ src_install() { "${D}"/usr/lib/dracut/modules.d/30ignition/ignition-setup.sh \ "${D}"/usr/lib/dracut/modules.d/30ignition/ignition-kargs-helper \ "${D}"/usr/lib/dracut/modules.d/30ignition/retry-umount.sh \ - "${D}"/usr/lib/dracut/modules.d/35torcx/torcx-profile-populate-generator \ "${D}"/usr/lib/dracut/modules.d/99setup-root/initrd-setup-root \ "${D}"/usr/lib/dracut/modules.d/99setup-root/initrd-setup-root-after-ignition \ "${D}"/usr/lib/dracut/modules.d/99setup-root/gpg-agent-wrapper \ From 89555ed2bd3d35d39d9d1dc5794f8d2d0e1790c3 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 10 Oct 2023 09:13:58 +0200 Subject: [PATCH 23/40] base OS sysexts: separate build script, inventory generation This change refactors base OS sysext builds to use a separate build script `build_library/sysext_prod_builder`, which is called from `build_library/prod_image_util.sh` when `build_image` runs. This allows for better separation of cleanup traps: prod image sysext builds need its own trap / cleanup function for temporary build directories and loopback mounts. Prod sysext builds properly generate lincense and SBOM information, and provide detailed file listings and disk space usage stats. - SBOM / licenses JSON now include all packages of the final image, i.e. a combined list of base image and all base OS sysexts. - Packages lists, files list and detailed files list include the sysext squashfs files for the base image, and separate sections with files / packages lists for each sysext. - Disk usage contains both final disk image usage as well as usage of each individual sysext squashfs. --- build_library/build_image_util.sh | 5 - build_library/prod_image_util.sh | 117 ++++++++++------------- build_library/sysext_prod_builder | 151 ++++++++++++++++++++++++++++++ build_sysext | 16 +++- 4 files changed, 213 insertions(+), 76 deletions(-) create mode 100755 build_library/sysext_prod_builder diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 7073da75d30..790c1a9556c 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -301,11 +301,6 @@ image_packages_implicit() { query_available_package "${pkg}" done < "${profile}/package.provided" fi - - # Include source packages of all sysext images installed on disk. - for docker_containerd_package in $(package_run_dependencies docker) $(package_run_dependencies containerd); do - query_available_package "${docker_containerd_package}" ; - done } # Generate a list of packages installed in an image. diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 6259f0dbdd2..09fa70ac591 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -52,58 +52,6 @@ extract_prod_gcc() { package_provided "${gcc}" } -# Create a sysext from a package and install it to the OS image. -# Conventions: -# - For each /, __pkginfo will be built. Can be used in subsequent calls -# to build dependent sysexts. -# - If ${BUILD_LIBRARY_DIR}/sysext_mangle__ exists it will be used as FS mangle script -# when building the sysext. -# -create_prod_sysext() { - local install_root="$1" - local base_image="$2" - local grp_pkg="$3" - local pkginfo="${4:-}" - - local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package - local pkginfo_opt="" - local manglefs_opt="" - - local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw" - - # Include previous sysexts' pkginfo if supplied - if [[ -n "${pkginfo}" ]] ; then - if [[ ! -f "${BUILD_DIR}/${pkginfo}" ]] ; then - die "Sysext build '${grp_pkg}': unable to find package info at '${BUILD_DIR}/${pkginfo}'." - fi - msg="${msg} w/ package info '${pkginfo}'" - pkginfo_opt="--base_pkginfo=${BUILD_DIR}/${pkginfo}" - fi - - # Include FS mangle script if present - if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then - manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" - msg="${msg}, FS mangle script 'sysext_mangle_${name}'" - fi - - info "${msg}." - - sudo "${SCRIPTS_DIR}/build_sysext" \ - --board="${BOARD}" \ - --image_builddir="${BUILD_DIR}" \ - --squashfs_base="${base_image}" \ - --generate_pkginfo \ - ${manglefs_opt} ${pkginfo_opt} \ - "${name}" "${grp_pkg}" - - sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext - sudo install -m 0644 -D "${BUILD_DIR}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/ - - sudo mkdir -p "${install_root}"/etc/extensions/ - sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw" -} -# -- - create_prod_image() { local image_name="$1" local disk_layout="$2" @@ -118,6 +66,7 @@ create_prod_image() { info "Building production image ${image_name}" local root_fs_dir="${BUILD_DIR}/rootfs" + local root_fs_sysexts_output_dir="${BUILD_DIR}/rootfs-included-sysexts" local image_contents="${image_name%.bin}_contents.txt" local image_contents_wtd="${image_name%.bin}_contents_wtd.txt" local image_packages="${image_name%.bin}_packages.txt" @@ -141,9 +90,31 @@ create_prod_image() { emerge_to_image "${root_fs_dir}" "${base_pkg}" run_ldconfig "${root_fs_dir}" run_localedef "${root_fs_dir}" + + local root_with_everything="${root_fs_dir}" + + # Call helper script for adding sysexts to the base OS. + # Helper will generate a rootfs dir with all packages (base OS and sysexts) included. + local root_sysext_mergedir="${BUILD_DIR}/rootfs-with-sysext-pkgs" + if [[ -n "${base_sysexts}" ]] ; then + "${BUILD_LIBRARY_DIR}/sysext_prod_builder" \ + "${BOARD}" "${BUILD_DIR}" "${root_fs_dir}" \ + "${root_sysext_mergedir}" \ + "${root_fs_sysexts_output_dir}" \ + "${base_sysexts}" + root_with_everything="${root_sysext_mergedir}" + fi + + + write_sbom "${root_with_everything}" "${BUILD_DIR}/${image_sbom}" + write_licenses "${root_with_everything}" "${BUILD_DIR}/${image_licenses}" + + if [[ -n "${base_sysexts}" ]] ; then + sudo rm -rf "${root_sysext_mergedir}" + fi + write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" - write_sbom "${root_fs_dir}" "${BUILD_DIR}/${image_sbom}" - write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" + insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" insert_extra_slsa "${root_fs_dir}" @@ -190,19 +161,9 @@ EOF # Remove source locale data, only need to ship the compiled archive. sudo rm -rf ${root_fs_dir}/usr/share/i18n/ - if [[ -n "${base_sysexts}" ]] ; then - local grp_pkg="" - local prev_pkginfo="" - for grp_pkg in ${base_sysexts//,/ }; do - create_prod_sysext "${root_fs_dir}"\ - "${BUILD_DIR}/${image_sysext_base}" \ - "${grp_pkg}" \ - "${prev_pkginfo}" - prev_pkginfo="${grp_pkg//\//_}_pkginfo.raw" - done - fi - # Finish image will move files from /etc to /usr/share/flatcar/etc. + # Note that image filesystem contents generated by finish_image will not + # include sysext contents (only the sysext squashfs files themselves). finish_image \ "${image_name}" \ "${disk_layout}" \ @@ -218,6 +179,30 @@ EOF "${image_initrd_contents_wtd}" \ "${image_disk_usage}" + # append sysext inventories to image contents files. + if [[ -n "${base_sysexts}" ]] ; then + local inventory_file="" image_basename="${image_name%.bin}" + + for inventory_file in "${image_contents}" "${image_contents_wtd}" "${image_disk_usage}" "${image_packages}" ; do + local suffix="${inventory_file/${image_basename}/}" sysext="" + + info "Processing '${inventory_file}'" + + for sysext in ${base_sysexts//,/ }; do + local name="${sysext//\//_}" + local sysext_inventory="${root_fs_sysexts_output_dir}/${name}${suffix}" + if [[ ! -f "${sysext_inventory}" ]] ; then + die "Sysext inventory file '${sysext//\//_}${suffix}' for '${inventory_file}' not found in '${root_fs_sysexts_output_dir}'" + fi + info "Adding sysext inventory '${name}${suffix}' to '${inventory_file}'" + { + echo -e "\n\n### Sysext ${name}.raw\n" + cat "${sysext_inventory}" + } >> "${BUILD_DIR}/${inventory_file}" + done + done + fi + # Upload local to_upload=( "${BUILD_DIR}/${image_contents}" diff --git a/build_library/sysext_prod_builder b/build_library/sysext_prod_builder new file mode 100755 index 00000000000..4f9aacb3ad6 --- /dev/null +++ b/build_library/sysext_prod_builder @@ -0,0 +1,151 @@ +#!/bin/bash +# Copyright (c) 2023 by the Flatcar Maintainers. +# Use of this source code is governed by the Apache 2.0 license. + +# Helper script for building OS images w/ sysexts included. +# Called by build_image -> prod_image_util.sh. +# This is a separate script mainly so we can trap EXIT and clean up our mounts +# without interfering with traps set by build_image. + +# We're in build_library/, script root is one up +SCRIPT_ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../"; pwd)" +. "${SCRIPT_ROOT}/common.sh" || exit 1 + +# Script must run inside the chroot +assert_inside_chroot +switch_to_strict_mode + +. "${BUILD_LIBRARY_DIR}/build_image_util.sh" || exit 1 + +# Create a sysext from a package and install it to the OS image. +# Conventions: +# - For each /, __pkginfo will be built. Can be used in subsequent calls +# to build dependent sysexts. +# - If ${BUILD_LIBRARY_DIR}/sysext_mangle__ exists it will be used as FS mangle script +# when building the sysext. +create_prod_sysext() { + local BOARD="$1" + local output_dir="$2" + local workdir="$3" + local base_sysext="$4" + local install_root="$5" + local grp_pkg="$6" + local pkginfo="${7:-}" + + local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package + local pkginfo_opt="" + local manglefs_opt="" + + local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw" + + # Include previous sysexts' pkginfo if supplied + if [[ -n "${pkginfo}" ]] ; then + if [[ ! -f "${output_dir}/${pkginfo}" ]] ; then + die "Sysext build '${grp_pkg}': unable to find package info at '${output_dir}/${pkginfo}'." + fi + msg="${msg} w/ package info '${pkginfo}'" + pkginfo_opt="--base_pkginfo=${output_dir}/${pkginfo}" + fi + + # Include FS mangle script if present + if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then + manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" + msg="${msg}, FS mangle script 'sysext_mangle_${name}'" + fi + + info "${msg}." + + sudo "${SCRIPTS_DIR}/build_sysext" \ + --board="${BOARD}" \ + --image_builddir="${workdir}/sysext-build" \ + --squashfs_base="${base_sysext}" \ + --generate_pkginfo \ + ${manglefs_opt} ${pkginfo_opt} \ + "${name}" "${grp_pkg}" + + sudo mv "${workdir}/sysext-build/${name}.raw" "${workdir}/sysext-build/${name}_pkginfo.raw" \ + "${workdir}/sysext-build/${name}"_*.txt "${output_dir}" + + sudo mkdir -p "${install_root}"/usr/share/flatcar/sysext + sudo install -m 0644 -D "${output_dir}/${name}.raw" "${install_root}"/usr/share/flatcar/sysext/ + + sudo mkdir -p "${install_root}"/etc/extensions/ + sudo ln -sf "/usr/share/flatcar/sysext/${name}.raw" "${install_root}/etc/extensions/${name}.raw" +} +# -- + +BOARD="$1" +BUILD_DIR="$2" +root_fs_dir="$3" + +merged_rootfs_dir="$4" +sysext_output_dir="$5" + +sysexts_list="$6" + +grp_pkg="" +prev_pkginfo="" +sysext_workdir="${BUILD_DIR}/prod-sysext-work" +sysext_mountdir="${BUILD_DIR}/prod-sysext-work/mounts" +sysext_base="${sysext_workdir}/base-os.squashfs" + +function cleanup() { + sudo umount "${sysext_mountdir}"/* || true + rm -rf "${sysext_workdir}" || true +} +# -- + +trap cleanup EXIT + +rm -rf "${sysext_workdir}" "${sysext_output_dir}" +mkdir "${sysext_workdir}" "${sysext_output_dir}" + +info "creating temporary base OS squashfs" +sudo mksquashfs "${root_fs_dir}" "${sysext_base}" -noappend + +# Build sysexts on top of root fs and mount sysexts' squashfs + pkginfo squashfs +# for combined overlay later. +prev_pkginfo="" +sysext_lowerdirs="${sysext_mountdir}/rootfs-lower" +for grp_pkg in ${sysexts_list//,/ }; do + create_prod_sysext "${BOARD}" \ + "${sysext_output_dir}" \ + "${sysext_workdir}" \ + "${sysext_base}" \ + "${root_fs_dir}"\ + "${grp_pkg}" \ + "${prev_pkginfo}" + name="${grp_pkg//\//_}" + + mkdir -p "${sysext_mountdir}/${name}" \ + "${sysext_mountdir}/${name}_pkginfo" + sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}.raw" \ + "${sysext_mountdir}/${name}" + sudo mount -rt squashfs -o loop,nodev "${sysext_output_dir}/${name}_pkginfo.raw" \ + "${sysext_mountdir}/${name}_pkginfo" + + sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}" + sysext_lowerdirs="${sysext_lowerdirs}:${sysext_mountdir}/${name}_pkginfo" + + prev_pkginfo="${name}_pkginfo.raw" +done + +# Mount the combined overlay (base OS, sysexts, and syset pkginfos) and copy a snapshot +# into the designated output dir for upper layers to process. +mkdir -p "${sysext_mountdir}/rootfs-lower" +sudo mount -rt squashfs -o loop,nodev "${sysext_base}" "${sysext_mountdir}/rootfs-lower" + +# Mount overlay for report generation +mkdir -p "${sysext_workdir}/.work" +mkdir -p "${sysext_mountdir}/rootfs-upper" +sudo mount -t overlay overlay \ + -o lowerdir="${sysext_lowerdirs}",upperdir="${sysext_mountdir}/rootfs-upper",workdir="${sysext_workdir}/.work" \ + "${sysext_mountdir}/rootfs-upper" + + +sudo rm -rf "${merged_rootfs_dir}" +sudo cp -a "${sysext_mountdir}/rootfs-upper" "${merged_rootfs_dir}" + + +cleanup +trap -- EXIT diff --git a/build_sysext b/build_sysext index bc7b0646e5d..95b00d49ce4 100755 --- a/build_sysext +++ b/build_sysext @@ -130,10 +130,11 @@ cleanup() { ) umount "${dirs[@]}" 2>/dev/null || true rm -rf "${dirs[@]}" || true - if [[ -d "${BUILD_DIR}/img-pkginfo" ]] ; then - umount "${BUILD_DIR}/img-pkginfo"/* 2>/dev/null || true - rm -rf "${BUILD_DIR}/img-pkginfo" || true + if [[ -d "${BUILD_DIR}/base-pkginfo" ]] ; then + umount "${BUILD_DIR}/base-pkginfo"/* 2>/dev/null || true + rm -rf "${BUILD_DIR}/base-pkginfo" || true fi + rm -rf "${BUILD_DIR}/img-pkginfo" } # Set up trap to execute cleanup() on script exit @@ -143,7 +144,7 @@ ARCH=$(_get_sysext_arch "${FLAGS_board}") cleanup # If we need to handle pkginfo squashfs files, create mount points under -# ${BUILD_DIR}/img-pkginfo, mount the squashfs images, and add the mount paths to +# ${BUILD_DIR}/base-pkginfo, mount the squashfs images, and add the mount paths to # the list of lowerdirs. pkginfo_lowerdirs="" if [[ -n "${FLAGS_base_pkginfo}" ]] ; then @@ -156,7 +157,7 @@ if [[ -n "${FLAGS_base_pkginfo}" ]] ; then fi pfile="$(basename "${ppath}")" - pmdir="${BUILD_DIR}/img-pkginfo/${pfile}" + pmdir="${BUILD_DIR}/base-pkginfo/${pfile}" mkdir -p "${pmdir}" mount -rt squashfs -o loop,nodev "${ppath}" "${pmdir}" pkginfo_lowerdirs="${pkginfo_lowerdirs}:${pmdir}" @@ -169,6 +170,7 @@ mount -rt squashfs -o loop,nodev "${FLAGS_squashfs_base}" "${BUILD_DIR}/fs-root" mkdir "${BUILD_DIR}/install-root" mkdir "${BUILD_DIR}/workdir" mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root${pkginfo_lowerdirs}",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root" + VERSION_BOARD=$(grep "^VERSION=" ${BUILD_DIR}/fs-root/usr/lib/os-release | cut -d = -f 2-) if [ "$VERSION_BOARD" != "$FLATCAR_VERSION" ]; then warn "Base squashfs version: $VERSION_BOARD" @@ -223,6 +225,10 @@ if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend fi +info "Writing ${SYSEXTNAME}_packages.txt" +ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root"\ + equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" + info "Removing non-/usr directories from sysext image" for entry in "${BUILD_DIR}/install-root"/*; do if [[ "${entry}" = */usr ]]; then From e85ca4984ce56d9f44627370f861dfabc961d875 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 12 Oct 2023 10:17:10 +0200 Subject: [PATCH 24/40] ci-automation: remove torcx artifact handling and tests --- ci-automation/README.md | 8 +-- ci-automation/ci-config.env | 1 - ci-automation/image.sh | 13 ++--- ci-automation/packages.sh | 50 ++--------------- ci-automation/push_pkgs.sh | 2 +- ci-automation/release.sh | 2 +- ci-automation/test.sh | 55 ++----------------- ci-automation/vendor-testing/aws.sh | 1 - ci-automation/vendor-testing/azure.sh | 1 - ci-automation/vendor-testing/digitalocean.sh | 1 - ci-automation/vendor-testing/equinix_metal.sh | 1 - ci-automation/vendor-testing/gce.sh | 1 - ci-automation/vendor-testing/openstack.sh | 1 - ci-automation/vendor-testing/qemu.sh | 1 - ci-automation/vendor-testing/qemu_update.sh | 1 - ci-automation/vendor-testing/vmware.sh | 1 - ci-automation/vendor_test.sh | 6 -- 17 files changed, 18 insertions(+), 128 deletions(-) diff --git a/ci-automation/README.md b/ci-automation/README.md index 9899580c6a0..aef5b76fa34 100644 --- a/ci-automation/README.md +++ b/ci-automation/README.md @@ -77,8 +77,8 @@ image_build amd64 4. Packages build (`packages.sh`): Build OS image packages and generate a new container image (containing both SDK and packages). 5. Packages are published and the generic OS image is built. 1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch. - 2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it. -6. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images. + 2. Image build (`image.sh`): Using the container from 3., build an OS image, and generate a new container image with everything in it. +6. VMs build (`vms.sh`). Using the packages+image container from 4., build vendor images. Results are vendor-specific OS images. ``` .---------. .------------. .--------. @@ -97,7 +97,7 @@ image_build amd64 | alpha-3499.0.0-dev23 `--------´ | | |`- sdk + OS packages -->| | | container image | - | | torcx manifest | + | | | | ______v_______ | | ( publish pkgs ) | | `------------´ | @@ -129,7 +129,7 @@ Testing is implemented in two layers: Testing relies on the SDK container and will use tools / test suites from the SDK. The low-level vendor / image specific script (layer 2. in the list above) runs inside the SDK. -Testing will use the vendor image published by `vms.sh` from buildcache, and the torcx manifest published by `packages.sh`. +Testing will use the vendor image published by `vms.sh` from buildcache. Additionally, a script library is provided (at `ci-automation/tapfile_helper_lib.sh`) to help handling `.tap` test result files produced by test runs. Library functions may be used to merge the result of multiple test runs (e.g. for multiple image types / vendors) into a single test result report. diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 39fdb46939c..52768fb8eef 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -28,7 +28,6 @@ CI_GIT_AUTHOR="flatcar-ci" CI_GIT_EMAIL="infra+ci@flatcar-linux.org" # build artifacts go here (in container) -CONTAINER_TORCX_ROOT="/home/sdk/trunk/src/build/torcx" CONTAINER_IMAGE_ROOT="/home/sdk/trunk/src/build/images" # Set it to "1" or "true" or "t" or "y" or "yes" to always run a full diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 92e9a22cd59..0cb5b7e1185 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -17,7 +17,7 @@ # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Flatcar packages container is available via build cache server # from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" -# or present locally. Container must contain binary packages and torcx artefacts. +# or present locally. Container must contain binary packages. # # INPUT: # @@ -37,7 +37,7 @@ # # OUTPUT: # -# 1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache. +# 1. OS image, dev container, and related artifacts pushed to buildcache. # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. @@ -84,17 +84,12 @@ function _image_build_impl() { official_arg="--noofficial" fi - local torcx_root_tar="torcx_root.tar.zst" apply_local_patches - copy_from_buildcache "images/${arch}/${vernum}/torcx/${torcx_root_tar}" . # build image and related artifacts ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ -v "${vernum}" \ - mkdir -p "${CONTAINER_IMAGE_ROOT}" "${CONTAINER_TORCX_ROOT}" - ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ - -v "${vernum}" \ - tar --zstd -xf "${torcx_root_tar}" -C "${CONTAINER_TORCX_ROOT}" + mkdir -p "${CONTAINER_IMAGE_ROOT}" ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ -v "${vernum}" \ ./set_official --board="${arch}-usr" "${official_arg}" @@ -103,7 +98,7 @@ function _image_build_impl() { ./build_image --board="${arch}-usr" --group="${channel}" \ --output_root="${CONTAINER_IMAGE_ROOT}" \ --only_store_compressed \ - --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container + prodtar container # copy resulting images + push to buildcache local images_out="images/" diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 1d4faf42c89..e6e984f7f38 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -41,8 +41,7 @@ # OUTPUT: # # 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages -# pushed to buildcache, and torcx_manifest.json pushed to "images/${arch}/${vernum}/" -# (for use with tests). +# pushed to buildcache (for use with tests). # 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. # 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache. @@ -81,38 +80,14 @@ function _packages_build_impl() { local vernum="${FLATCAR_VERSION}" local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" local packages_container="flatcar-packages-${arch}-${docker_vernum}" - local torcx_pkg_url="https://${BUILDCACHE_SERVER}/images/${arch}/${vernum}/torcx" source sdk_lib/sdk_container_common.sh - if is_official "${vernum}"; then - # A channel returned by get_git_channel should not ever be - # "developer" here, because it's an official build done from - # one of the maintenance branches. So if the channel happens - # to be "developer", then you are doing it wrong (releasing - # from the main branch?). - torcx_pkg_url="https://$(get_git_channel).release.flatcar-linux.net/${arch}-usr/${vernum}/torcx" - fi - apply_local_patches - # Build packages; store packages and torcx output in container - ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \ - -C "${sdk_image}" \ - mkdir -p "${CONTAINER_TORCX_ROOT}" - ./run_sdk_container -n "${packages_container}" -v "${vernum}" \ - -C "${sdk_image}" \ - ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CONTAINER_TORCX_ROOT}" \ - --torcx_extra_pkg_url="${torcx_pkg_url}" - - # copy torcx manifest and docker tarball for publishing - local torcx_tmp="__build__/torcx_tmp" - rm -rf "${torcx_tmp}" - mkdir "${torcx_tmp}" - ./run_sdk_container -n "${packages_container}" -v "${vernum}" \ + # Build packages; store packages in container + ./run_sdk_container -x ./ci-cleanup.sh -n "${packages_container}" -v "${vernum}" \ -C "${sdk_image}" \ - cp -r "${CONTAINER_TORCX_ROOT}/" \ - "${torcx_tmp}" + ./build_packages --board="${arch}-usr" # run_sdk_container updates the version file, use that version from here on source sdk_container/.repo/manifests/version.txt @@ -122,22 +97,5 @@ function _packages_build_impl() { # generate image + push to build cache docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}" - - # publish torcx output root for consumption by build_image - local torcx_root_tar="torcx_root.tar.zst" - tar --zstd -cpf "${torcx_root_tar}" -C "${torcx_tmp}/torcx" . - copy_to_buildcache "images/${arch}/${vernum}/torcx" "${torcx_root_tar}" - - # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later. - create_digests "${SIGNER}" \ - "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \ - "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz - sign_artifacts "${SIGNER}" \ - "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* \ - "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz* - copy_to_buildcache "images/${arch}/${vernum}/torcx" \ - "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json"* - copy_to_buildcache "images/${arch}/${vernum}/torcx" \ - "${torcx_tmp}/torcx/pkgs/${arch}-usr/docker/"*/*.torcx.tgz* } # -- diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh index 86439c599b7..0cb1a0187cf 100644 --- a/ci-automation/push_pkgs.sh +++ b/ci-automation/push_pkgs.sh @@ -19,7 +19,7 @@ # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Flatcar packages container is available via build cache server # from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" -# or present locally. Container must contain binary packages and torcx artifacts. +# or present locally. Container must contain binary packages. # # INPUT: # diff --git a/ci-automation/release.sh b/ci-automation/release.sh index 930fd9d9044..90fb1c59db2 100644 --- a/ci-automation/release.sh +++ b/ci-automation/release.sh @@ -16,7 +16,7 @@ # 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container. -# 4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache +# 4. Vendor image to run tests for are available on buildcache # ( images/[ARCH]/[FLATCAR_VERSION]/ ) # 5. SDK container is either # - available via ghcr.io/flatcar/flatcar-sdk-[ARCH]:[VERSION] (official SDK release) diff --git a/ci-automation/test.sh b/ci-automation/test.sh index c3b14fe2f36..bdddb4a3b43 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -19,7 +19,7 @@ # 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt # 2. Scripts repo version tag of OS image version to be built is available and checked out. # 3. Mantle container docker image reference is stored in sdk_container/.repo/manifests/mantle-container. -# 4. Vendor image and torcx docker tarball + manifest to run tests for are available on buildcache +# 4. Vendor image to run tests for are available on buildcache # ( images/[ARCH]/[FLATCAR_VERSION]/ ) # # INPUT: @@ -54,14 +54,10 @@ # # Vendor scripts are provided with their own sub-directory and are expected to CD into there before # creating any artifacts (see vendor script argument 1 below). -# The torcx manifest is supplied in -# ../ -# relative to the vendor sub-directory. The manifest is updated to include a URL pointing to the docker -# torcx tarball on the build cache (for the docker.torcx-manifest-pkgs test). # # Vendor specific scripts are called with the following positional arguments: # 1 - Toplevel tests directory -# It contains some additional files needed for running the tests (like torcx manifest or file with channel information). +# It contains some additional files needed for running the tests (like file with channel information). # 2 - Working directory for the tests. # The vendor script is expected to keep all artifacts it produces in that directory. # 3 - Architecture to test. @@ -74,46 +70,6 @@ # script would need to make anyway. For more information, please refer # to the vendor_test.sh file. -# Download torcx manifest and modify URLs pointing to the origin -# server to point to the build cache. This is because the tests for -# releases are run before artifacts are uploaded to the origin -# server. This would make kola's docker.torcx-manifest-pkgs test to -# fail. -function __prepare_torcx() { - local arch="$1" - local vernum="$2" - local workdir="$3" - - copy_from_buildcache "images/${arch}/${vernum}/torcx/torcx_manifest.json" "${workdir}" - - # Change URLs from: - # - # https://${channel}.release.flatcar-linux.net/${arch}-usr/${vernum}/torcx/… - # - # to: - # - # https://bincache.flatcar-linux.net/images/${arch}/${vernum}/torcx/… - # - # This is done in two parts - replacing host part and arch part. - # - # Replace 'https://${channel}.release.flatcar-linux.net/' with - # 'https://bincache.flatcar-linux.net/' matching the initial "url" - # JSON key too. - local host_match='\("url":\s*"https://\)[a-z]\+\.release\([^/]\+/\)' - local host_replace='\1bincache\2' - # Replace '${arch}-usr/` part with 'images/${arch}/'. - local arch_match='\([a-z0-9]\+\)-usr/' - local arch_replace='images/\3/' - sed \ - -e "s#${host_match}${arch_match}#${host_replace}${arch_replace}#g" \ - "${workdir}/torcx_manifest.json" \ - >"${workdir}/torcx_manifest_new.json" - - mv "${workdir}/torcx_manifest.json" "${workdir}/torcx_manifest.json.original" - mv "${workdir}/torcx_manifest_new.json" "${workdir}/torcx_manifest.json" -} -# -- - function __escape_multiple() { local out_array_arg_name="${1}"; shift # rest are args to be escape and appended into the array named @@ -176,9 +132,6 @@ function _test_run_impl() { local mantle_ref mantle_ref=$(cat sdk_container/.repo/manifests/mantle-container) - # Make the torcx artifacts available to test implementation - __prepare_torcx "${arch}" "${vernum}" "${work_dir}" - local tap_merged_summary="results-${image}" local tap_merged_detailed="results-${image}-detailed" local retry="" @@ -214,7 +167,7 @@ function _test_run_impl() { # determine success based on test results (tapfile). set +e touch sdk_container/.env - docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ + docker run --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ -w /work -v "$PWD":/work "${mantle_ref}" \ bash -c "git config --global --add safe.directory /work && \ source sdk_container/.env && \ @@ -223,7 +176,7 @@ function _test_run_impl() { rm -f "${work_dir}/first_run" # Note: git safe.directory is not set in this run as it does not use git - docker run --pull always --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ + docker run --rm --name="${container_name}" --privileged --net host -v /dev:/dev \ -w /work -v "$PWD":/work "${mantle_ref}" \ ci-automation/test_update_reruns.sh \ "${arch}" "${vernum}" "${image}" "${retry}" \ diff --git a/ci-automation/vendor-testing/aws.sh b/ci-automation/vendor-testing/aws.sh index f9e3c34f068..f33f668b8e5 100755 --- a/ci-automation/vendor-testing/aws.sh +++ b/ci-automation/vendor-testing/aws.sh @@ -58,7 +58,6 @@ run_kola_tests() { --aws-type="${instance_type}" \ --aws-iam-profile="${AWS_IAM_PROFILE}" \ --tapfile="${instance_tapfile}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ "${@}" } diff --git a/ci-automation/vendor-testing/azure.sh b/ci-automation/vendor-testing/azure.sh index 6fc84ff3034..bc7adf16ae6 100755 --- a/ci-automation/vendor-testing/azure.sh +++ b/ci-automation/vendor-testing/azure.sh @@ -53,7 +53,6 @@ run_kola_tests() { --azure-image-file="${AZURE_IMAGE_NAME}" \ --azure-location="${AZURE_LOCATION}" \ --azure-identity \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --tapfile="${instance_tapfile}" \ --azure-size="${instance_type}" \ --azure-hyper-v-generation="${hyperv_gen}" \ diff --git a/ci-automation/vendor-testing/digitalocean.sh b/ci-automation/vendor-testing/digitalocean.sh index 3ca82ac2e46..14e100ed291 100755 --- a/ci-automation/vendor-testing/digitalocean.sh +++ b/ci-automation/vendor-testing/digitalocean.sh @@ -51,7 +51,6 @@ timeout --signal=SIGQUIT 4h\ --platform=do \ --channel="${CIA_CHANNEL}" \ --tapfile="${CIA_TAPFILE}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ "${@}" set +x diff --git a/ci-automation/vendor-testing/equinix_metal.sh b/ci-automation/vendor-testing/equinix_metal.sh index 5f4440fd122..8bab8541105 100755 --- a/ci-automation/vendor-testing/equinix_metal.sh +++ b/ci-automation/vendor-testing/equinix_metal.sh @@ -36,7 +36,6 @@ run_kola_tests() { --platform=equinixmetal \ --tapfile="${instance_tapfile}" \ --parallel="${EQUINIXMETAL_PARALLEL}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --equinixmetal-image-url="${BASE_URL}/${EQUINIXMETAL_IMAGE_NAME}" \ --equinixmetal-installer-image-kernel-url="${BASE_URL}/${PXE_KERNEL_NAME}" \ --equinixmetal-installer-image-cpio-url="${BASE_URL}/${PXE_IMAGE_NAME}" \ diff --git a/ci-automation/vendor-testing/gce.sh b/ci-automation/vendor-testing/gce.sh index 2ab0fd1284d..ec26fbe57fb 100755 --- a/ci-automation/vendor-testing/gce.sh +++ b/ci-automation/vendor-testing/gce.sh @@ -62,7 +62,6 @@ run_kola_tests() { --platform=gce \ --channel="${CIA_CHANNEL}" \ --tapfile="${instance_tapfile}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ "${@}" } diff --git a/ci-automation/vendor-testing/openstack.sh b/ci-automation/vendor-testing/openstack.sh index caaa7fe3b1d..4efa6d49071 100755 --- a/ci-automation/vendor-testing/openstack.sh +++ b/ci-automation/vendor-testing/openstack.sh @@ -49,7 +49,6 @@ timeout --signal=SIGQUIT 2h kola run \ --parallel="${OPENSTACK_PARALLEL}" \ --tapfile="${CIA_TAPFILE}" \ --channel="${CIA_CHANNEL}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --basename="${kola_test_basename}" \ --platform=openstack \ --openstack-network=public \ diff --git a/ci-automation/vendor-testing/qemu.sh b/ci-automation/vendor-testing/qemu.sh index bd1763656a2..7b426889e41 100755 --- a/ci-automation/vendor-testing/qemu.sh +++ b/ci-automation/vendor-testing/qemu.sh @@ -64,7 +64,6 @@ kola run \ --qemu-bios="${bios}" \ --qemu-image="${QEMU_IMAGE_NAME}" \ --tapfile="${CIA_TAPFILE}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \ "${devcontainer_opts[@]}" \ "${@}" diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index d4cf2d8105c..5232b099383 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -115,7 +115,6 @@ run_kola_tests() { --qemu-bios="${bios}" \ --qemu-image="${image}" \ --tapfile="${instance_tapfile}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --update-payload="${QEMU_UPDATE_PAYLOAD}" \ ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \ "${tests[@]}" diff --git a/ci-automation/vendor-testing/vmware.sh b/ci-automation/vendor-testing/vmware.sh index 399fbfc897f..ce0ca1eeff5 100755 --- a/ci-automation/vendor-testing/vmware.sh +++ b/ci-automation/vendor-testing/vmware.sh @@ -60,7 +60,6 @@ sudo timeout --signal=SIGQUIT 2h kola run \ --platform=esx \ --tapfile="${CIA_TAPFILE}" \ --parallel="${VMWARE_ESX_PARALLEL}" \ - --torcx-manifest="${CIA_TORCX_MANIFEST}" \ --esx-config-file "${config_file}" \ --esx-ova-path "${VMWARE_ESX_IMAGE_NAME}" \ "${@}" diff --git a/ci-automation/vendor_test.sh b/ci-automation/vendor_test.sh index f0d657e36d9..60081a67727 100644 --- a/ci-automation/vendor_test.sh +++ b/ci-automation/vendor_test.sh @@ -47,10 +47,6 @@ # It's either "release" or "developer", based on the CIA_VERNUM # variable. # -# CIA_TORCX_MANIFEST: -# Path to the Torcx manifest. Usually passed to kola through the -# --torcx-manifest parameter. -# # CIA_FIRST_RUN: # 1 if this is a first run, 0 if it is a rerun of failed tests. # @@ -91,7 +87,6 @@ fi # Make these paths absolute to avoid problems when changing # directories. ciavts_tapfile="${PWD}/${ciavts_work_dir}/${ciavts_tapfile}" -ciavts_torcx_manifest="${PWD}/${ciavts_main_work_dir}/torcx_manifest.json" ciavts_first_run=0 if [[ -f "${ciavts_main_work_dir}/first_run" ]]; then @@ -109,7 +104,6 @@ CIA_CHANNEL="${ciavts_channel}" CIA_TESTSCRIPT="${ciavts_testscript}" CIA_GIT_VERSION="${ciavts_git_version}" CIA_BUILD_TYPE="${ciavts_type}" -CIA_TORCX_MANIFEST="${ciavts_torcx_manifest}" CIA_FIRST_RUN="${ciavts_first_run}" # Unset all variables with ciavts_ prefix now. From 6f651a298c0080c896527d4ef73b56a5dbb3da2b Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 13 Oct 2023 16:41:31 +0200 Subject: [PATCH 25/40] build scripts: separate txt inventory; official naming for sysexts Signed-off-by: Thilo Fromm --- build_image | 4 +- build_library/prod_image_util.sh | 42 ++++++++----------- ...inerd => sysext_mangle_containerd-flatcar} | 0 ...rs_docker => sysext_mangle_docker-flatcar} | 0 build_library/sysext_prod_builder | 15 ++++--- 5 files changed, 29 insertions(+), 32 deletions(-) rename build_library/{sysext_mangle_app-containers_containerd => sysext_mangle_containerd-flatcar} (100%) rename build_library/{sysext_mangle_app-containers_docker => sysext_mangle_docker-flatcar} (100%) diff --git a/build_image b/build_image index a10a410f1d7..6cd84d91b65 100755 --- a/build_image +++ b/build_image @@ -33,8 +33,8 @@ DEFINE_string base_pkg "coreos-base/coreos" \ "The base portage package to base the build off of (only applies to prod images)" DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \ "The base portage package to base the build off of (only applies to dev containers)" -DEFINE_string base_sysexts "app-containers/containerd,app-containers/docker" \ - "Comma-separated list of packages to build into sysexts and include with OS image and update payload. Must be in order of dependencies, base sysexts come first." +DEFINE_string base_sysexts "containerd-flatcar:app-containers/containerd,docker-flatcar:app-containers/docker" \ + "Comma-separated list of name:package - build 'package' into sysext 'name', and include with OS image and update payload. Must be in order of dependencies, base sysexts come first." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string disk_layout "" \ diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 09fa70ac591..1f8029dc8c1 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -179,30 +179,6 @@ EOF "${image_initrd_contents_wtd}" \ "${image_disk_usage}" - # append sysext inventories to image contents files. - if [[ -n "${base_sysexts}" ]] ; then - local inventory_file="" image_basename="${image_name%.bin}" - - for inventory_file in "${image_contents}" "${image_contents_wtd}" "${image_disk_usage}" "${image_packages}" ; do - local suffix="${inventory_file/${image_basename}/}" sysext="" - - info "Processing '${inventory_file}'" - - for sysext in ${base_sysexts//,/ }; do - local name="${sysext//\//_}" - local sysext_inventory="${root_fs_sysexts_output_dir}/${name}${suffix}" - if [[ ! -f "${sysext_inventory}" ]] ; then - die "Sysext inventory file '${sysext//\//_}${suffix}' for '${inventory_file}' not found in '${root_fs_sysexts_output_dir}'" - fi - info "Adding sysext inventory '${name}${suffix}' to '${inventory_file}'" - { - echo -e "\n\n### Sysext ${name}.raw\n" - cat "${sysext_inventory}" - } >> "${BUILD_DIR}/${inventory_file}" - done - done - fi - # Upload local to_upload=( "${BUILD_DIR}/${image_contents}" @@ -220,6 +196,24 @@ EOF "${BUILD_DIR}/${image_sysext_base}" ) + # append sysext inventories to uploads + if [[ -n "${base_sysexts}" ]] ; then + local inventory_file="" image_basename="${image_name%.bin}" + + for inventory_file in "${image_contents}" "${image_contents_wtd}" "${image_disk_usage}" "${image_packages}" ; do + local suffix="${inventory_file/${image_basename}/}" sysext="" + + for sysext in ${base_sysexts//,/ }; do + local name="${sysext%:*}" + local sysext_inventory="${root_fs_sysexts_output_dir}/${name}${suffix}" + if [[ ! -f "${sysext_inventory}" ]] ; then + die "Sysext inventory file '${name}${suffix}' for '${inventory_file}' not found in '${root_fs_sysexts_output_dir}'" + fi + to_upload+=( "${sysext_inventory}" ) + done + done + fi + local files_to_evaluate=( "${BUILD_DIR}/${image_name}" ) declare -a compressed_images declare -a extra_files diff --git a/build_library/sysext_mangle_app-containers_containerd b/build_library/sysext_mangle_containerd-flatcar similarity index 100% rename from build_library/sysext_mangle_app-containers_containerd rename to build_library/sysext_mangle_containerd-flatcar diff --git a/build_library/sysext_mangle_app-containers_docker b/build_library/sysext_mangle_docker-flatcar similarity index 100% rename from build_library/sysext_mangle_app-containers_docker rename to build_library/sysext_mangle_docker-flatcar diff --git a/build_library/sysext_prod_builder b/build_library/sysext_prod_builder index 4f9aacb3ad6..a1db9acd7d1 100755 --- a/build_library/sysext_prod_builder +++ b/build_library/sysext_prod_builder @@ -29,14 +29,14 @@ create_prod_sysext() { local workdir="$3" local base_sysext="$4" local install_root="$5" - local grp_pkg="$6" - local pkginfo="${7:-}" + local name="$6" + local grp_pkg="$7" + local pkginfo="${8:-}" - local name="${grp_pkg//\//_}" # some-group/some-package => some-group_some-package local pkginfo_opt="" local manglefs_opt="" - local msg="Creating sysext '${grp_pkg}' ==> ${name}.raw" + local msg="Installing ${grp_pkg}' in sysext ${name}.raw" # Include previous sysexts' pkginfo if supplied if [[ -n "${pkginfo}" ]] ; then @@ -107,15 +107,18 @@ sudo mksquashfs "${root_fs_dir}" "${sysext_base}" -noappend # for combined overlay later. prev_pkginfo="" sysext_lowerdirs="${sysext_mountdir}/rootfs-lower" -for grp_pkg in ${sysexts_list//,/ }; do +for sysext in ${sysexts_list//,/ }; do + # format is ":/" + name="${sysext%:*}" + grp_pkg="${sysext#*:}" create_prod_sysext "${BOARD}" \ "${sysext_output_dir}" \ "${sysext_workdir}" \ "${sysext_base}" \ "${root_fs_dir}"\ + "${name}" \ "${grp_pkg}" \ "${prev_pkginfo}" - name="${grp_pkg//\//_}" mkdir -p "${sysext_mountdir}/${name}" \ "${sysext_mountdir}/${name}_pkginfo" From 35779f1ca3048388993eadede6931b83310aea31 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 13 Oct 2023 17:59:49 +0200 Subject: [PATCH 26/40] run_local_tests.sh: script for running tests on local builds This change adds run_local_tests.sh, a script to run tests on local builds. It's a comfort wrapper around ci-automation scripts and uses the latest local build. Signed-off-by: Thilo Fromm --- run_local_tests.sh | 108 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100755 run_local_tests.sh diff --git a/run_local_tests.sh b/run_local_tests.sh new file mode 100755 index 00000000000..f13c2e6bff7 --- /dev/null +++ b/run_local_tests.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Copyright (c) 2023 The Flatcar Maintainers. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# QoL wrapper around ci-automation test.sh for running local tests of qemu_uefi image. +# The devcontainer tests will be skipped since these require a valid commit ref in +# the upstream scripts repo. +# +# Requirements: +# - Docker (for running the Mantle container). +# +# Prerequisites: +# - Flatcar OS image and qemu uefi code to be tested in +# __build__/images/images/amd64-usr/latest/ +# +# This script is intended to be run after building a qemu_uefi image with the SDK container: +# ./build_packages +# ./build_image +# ./image_to_vm.sh --from=../build/images/amd64-usr/latest/ --format=qemu_uefi --image_compression_formats none +# Then, EXIT the SDK container (or run this on a different terminal): +# ./run_local_tests.sh +# +# Optional prerequisites: +# - Custom Mantle container image / version in sdk_container/.repo/manifests/mantle-container. +# This comes in handy if you've built a local mantle/kola which you want to test. +# Just edit the file and put in the whole containerr image name and version. +# +# Output: +# results reports: +# - results-qemu_uefi-detailed.md +# - results-qemu_uefi-detailed.tap +# - results-qemu_uefi.md +# - results-qemu_uefi.tap +# - results-qemu_update-detailed.md +# - results-qemu_update-detailed.tap +# - results-qemu_update.md +# - results-qemu_update.tap +# +# +# - Detailed test run output will reside below __TESTS__/qemu-uefi + +function set_vars() { + local arch="${1}" + local parallel="${2}" + + # Read by the mantle container. + # The local directory ("pwd") will be mounted to /work/ in the container. + cat > sdk_container/.env < Date: Mon, 16 Oct 2023 12:19:26 +0200 Subject: [PATCH 27/40] .github: removed torcx artifact handling from github CI Signed-off-by: Thilo Fromm --- .github/workflows/ci.yaml | 28 ++----------------- .github/workflows/run-kola-tests.yaml | 40 ++------------------------- 2 files changed, 5 insertions(+), 63 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d021dd0af7a..55126730322 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -100,14 +100,8 @@ jobs: [ -z "${{ inputs.image_formats }}" ] || IMAGE_FORMATS="${{ inputs.image_formats }}" echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV - # Artifact root for images and torcx tarball as seen from within the container + # Artifact root for images as seen from within the container echo "CI_CONTAINER_ARTIFACT_ROOT=/home/sdk/trunk/src/scripts/artifacts" >> $GITHUB_ENV - echo "CI_CONTAINER_TORCX_ROOT=/home/sdk/trunk/src/scripts/artifacts/torcx" >> $GITHUB_ENV - mkdir -p artifacts/torcx - - # Placeholder URL for run-kola-tests.yaml, "Extract artifacts" step which will replace - # this with its IP address. - echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV if [ -n "${{ inputs.custom_sdk_version }}" ] ; then echo "CUSTOM_SDK_VERSION=${{ inputs.custom_sdk_version }}" >> $GITHUB_ENV @@ -146,9 +140,7 @@ jobs: # which will be re-used by subsequent build steps. ./run_sdk_container -n "${container_name}" -v "${version}" \ -C "${sdk_image}" \ - ./build_packages --board="${arch}-usr" \ - --torcx_output_root="${CI_CONTAINER_TORCX_ROOT}" \ - --torcx_extra_pkg_url="${TORCX_TESTS_PACKAGE_URL}" + ./build_packages --board="${arch}-usr" # Create binpkgs tarball for archiving as artifact later ./run_sdk_container -n "${container_name}" \ @@ -193,7 +185,7 @@ jobs: ./run_sdk_container -n "${container_name}" \ ./build_image --board="${arch}-usr" --group="${channel}" \ --output_root="${CI_CONTAINER_ARTIFACT_ROOT}" \ - --torcx_root="${CI_CONTAINER_TORCX_ROOT}" prodtar container + prodtar container - name: Generate reports shell: bash @@ -291,12 +283,6 @@ jobs: mv * ../../images/ ) - # create a tarball for torcx package + JSON file because upload-artifacts cannot handle filenames containing colons - # (such as "docker:20.10.torcx.tgz") - mv artifacts/torcx/${arch}-usr/latest/torcx_manifest.json artifacts/torcx/pkgs/ - tar -C artifacts/torcx/pkgs/ -cvf torcx.tar . - - - name: Upload binpkgs uses: actions/upload-artifact@v3 with: @@ -336,14 +322,6 @@ jobs: path: | scripts/artifacts/images/flatcar_developer_container* - - name: Upload torcx tarball - uses: actions/upload-artifact@v3 - with: - retention-days: 7 - name: ${{ matrix.arch }}-torcx - path: | - scripts/torcx.tar - - name: Upload reports uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 8c5c0616bbb..816ad60e0c8 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -99,12 +99,6 @@ jobs: with: name: ${{ matrix.arch }}-devcontainer - - name: Download torcx tarball - if: ${{ !inputs.workflow_run_id }} - uses: actions/download-artifact@v3 - with: - name: ${{ matrix.arch }}-torcx - - name: Download binpkgs from other workflow uses: gabriel-samfira/action-download-artifact@v5 if: ${{ inputs.workflow_run_id }} @@ -141,15 +135,6 @@ jobs: run_id: ${{ inputs.workflow_run_id }} name: ${{ matrix.arch }}-devcontainer - - name: Download torcx tarball from other workflow - uses: gabriel-samfira/action-download-artifact@v5 - if: ${{ inputs.workflow_run_id }} - with: - workflow: ${{ inputs.workflow_name_or_id }} - workflow_conclusion: success - run_id: ${{ inputs.workflow_run_id }} - name: ${{ matrix.arch }}-torcx - - name: Extract artifacts shell: bash run: | @@ -157,8 +142,8 @@ jobs: set -x set -euo pipefail - # Set up a webserver for devcontainer and torcx tests. - # The respective tests will download devcontainer and torcx tarball via http. + # Set up a webserver for devcontainer tests. + # The respective tests will download devcontainer via http. # The devcontainer test will then run a build # which will download and install binpkgs into the dev container. # For the sake of that test we will serve both via a temporary local web server. @@ -174,19 +159,6 @@ jobs: mv flatcar_developer_container* ${TESTS_WEBSERVER_WEBROOT} tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf binpkgs.tar - tar -C ${TESTS_WEBSERVER_WEBROOT} -xvf torcx.tar - - # Move torcx package into plain webroot - # (path consists of ///:.torcx.tar.gz) - mv "${TESTS_WEBSERVER_WEBROOT}/${{ matrix.arch }}-usr"/*/*/*.torcx.tgz \ - "${TESTS_WEBSERVER_WEBROOT}" - - # Update torcx.json's http URL to point to the webserver IP. - # ci.yaml defines the "localhost" placeholder in its "Set Environment" step. - sed -i "s,http://localhost:12345,http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}," \ - "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" - cat "${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" - # Extract the generic image we'll use for qemu tests. # Note that the qemu[_uefi] tests use the generic image instead of the # qemu vendor VM image ("Astronaut: [...] Always have been."). @@ -221,14 +193,6 @@ jobs: source ci-automation/test.sh - # Provide our own torcx prepare function so we use our local manifest json. - # This is called by test_run below. - function __prepare_torcx() { - shift; shift # no need for arch or vernum - local destdir="$1" - cp "../${TESTS_WEBSERVER_WEBROOT}/torcx_manifest.json" "${destdir}" - } - PARALLEL_ARCH=10 cat > sdk_container/.env < Date: Tue, 17 Oct 2023 16:14:36 +0200 Subject: [PATCH 28/40] vendor-testing/qemu_update.sh: make update payload configurable This change makes QEMU_UPDATE_PAYLOAD configurable via ci-automation/settings.env where it was hard-wired before. The change also fixes fall-out in qemu_update.sh by ensuring a local tmp directory is created before it is used by the test. Signed-off-by: Thilo Fromm --- ci-automation/ci-config.env | 2 +- ci-automation/vendor-testing/qemu_update.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 52768fb8eef..8fce6b67cb4 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -68,7 +68,7 @@ QEMU_UEFI_BIOS="${QEMU_UEFI_BIOS:-flatcar_production_qemu_uefi_efi_code.fd}" # Update payload for the qemu_update.sh test. # The default path set below is relative to TEST_WORK_DIR -QEMU_UPDATE_PAYLOAD="tmp/flatcar_test_update.gz" +QEMU_UPDATE_PAYLOAD="${QEMU_UPDATE_PAYLOAD:-tmp/flatcar_test_update.gz}" # Devcontainer settings for isolated / local testing w/o a remote # devcontainer server and/or binhost. diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh index 5232b099383..986685c9185 100755 --- a/ci-automation/vendor-testing/qemu_update.sh +++ b/ci-automation/vendor-testing/qemu_update.sh @@ -24,10 +24,10 @@ if [ "$*" != "" ] && [ "$*" != "*" ] && [[ "$*" != *"cl.update."* ]]; then fi mkdir -p "$(dirname ${QEMU_UPDATE_PAYLOAD})" +mkdir -p tmp if [ -f "${QEMU_UPDATE_PAYLOAD}" ] ; then echo "++++ ${CIA_TESTSCRIPT}: Using existing ${QEMU_UPDATE_PAYLOAD} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++" else - # TODO: Change the GitHub Action to provide this artifact and detect that case here and skip the bincache download if ! curl --head -o /dev/null -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://bincache.flatcar-linux.net/images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update.gz"; then echo "1..1" > "${CIA_TAPFILE}" echo "ok - skipped qemu update tests" >> "${CIA_TAPFILE}" From 919d3686883397e2d892b9fa8eddca172e022155 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 17 Oct 2023 16:18:31 +0200 Subject: [PATCH 29/40] mantle-container: bump image ref to support torcx removal This change bumps the image ref of the mantle container to ghcr.io/flatcar/mantle:git-20a2f8ffee8c8a1a042b1da99f0f59312110f285. This version includes 2 PRs (https://github.com/flatcar/mantle/pull/465 and https://github.com/flatcar/mantle/pull/466) which add support for sysext docker / torcx removal in the OS image. Signed-off-by: Thilo Fromm --- changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md diff --git a/changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md b/changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md new file mode 100644 index 00000000000..41f093195a9 --- /dev/null +++ b/changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md @@ -0,0 +1 @@ +- SDK: bump Mantle / Kola container image version to include torcx removal / sysext tests ([Mantle#466](https://github.com/flatcar/mantle/pull/466), [Mantle#465](https://github.com/flatcar/mantle/pull/465)) From 428503083f96c2624e3075fac0c6bd6358618736 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 17 Oct 2023 18:13:18 +0200 Subject: [PATCH 30/40] run_sdk_container: Add -U flag to use the version file unchanged This change adds a -U flag to run_sdk_container. If provided, the script will not regenerate version.txt but instead use the existing file as-is. Signed-off-by: Thilo Fromm --- run_sdk_container | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/run_sdk_container b/run_sdk_container index b34b7958c13..4bdbf607837 100755 --- a/run_sdk_container +++ b/run_sdk_container @@ -45,11 +45,13 @@ usage() { echo " -C - Use an entirely custom container image instead of the SDK's" echo " $sdk_container_common_registry/flatcar-sdk-[ARCH]:[SDK VERSION]" echo " Useful for CI." + echo " -U Do not update the versionfile. Instead, use the version from the versionfile as-is." echo " -h Print this help." echo } # -- +update_versionfile="true" while [ 0 -lt $# ] ; do case "$1" in -h) usage; exit 0;; @@ -62,6 +64,10 @@ while [ 0 -lt $# ] ; do --rm) remove=true; shift;; -x) cleanup="$2"; shift; shift;; -C) custom_image="$2"; shift; shift;; + -U) sdk_version="$(get_sdk_version_from_versionfile)" + os_version="$(get_version_from_versionfile)" + update_versionfile="false" + shift;; *) break;; esac done @@ -73,7 +79,9 @@ else container_image_name="$sdk_container_common_registry/flatcar-sdk-${arch}:${docker_sdk_vernum}" fi -create_versionfile "$sdk_version" "$os_version" +if [[ "${update_versionfile}" == true ]] ; then + create_versionfile "$sdk_version" "$os_version" +fi if [ -z "$name" ] ; then docker_sdk_vernum="$(vernum_to_docker_image_version "${sdk_version}")" From 6512ef04cacd4cdb97c0910c0a7af7e15474f041 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Tue, 17 Oct 2023 18:27:12 +0200 Subject: [PATCH 31/40] overlay coreos-base/update_engine: remove torcx sanity check Include PR https://github.com/flatcar/update_engine/pull/30 to un-break updates when torcx was removed in favour of sysext. Signed-off-by: Thilo Fromm --- changelog/changes/2023-10-17-update-engine-torcx-removal.md | 1 + ...engine-0.4.10-r15.ebuild => update_engine-0.4.10-r16.ebuild} | 0 .../coreos-base/update_engine/update_engine-9999.ebuild | 2 +- 3 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changelog/changes/2023-10-17-update-engine-torcx-removal.md rename sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/{update_engine-0.4.10-r15.ebuild => update_engine-0.4.10-r16.ebuild} (100%) diff --git a/changelog/changes/2023-10-17-update-engine-torcx-removal.md b/changelog/changes/2023-10-17-update-engine-torcx-removal.md new file mode 100644 index 00000000000..3e04769cec3 --- /dev/null +++ b/changelog/changes/2023-10-17-update-engine-torcx-removal.md @@ -0,0 +1 @@ +- update_engine: remove torcx sanity checks because the OS image removed torcx ([update_engine#30](https://github.com/flatcar/update_engine/pull/30)) diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-0.4.10-r15.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-0.4.10-r16.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-0.4.10-r15.ebuild rename to sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-0.4.10-r16.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-9999.ebuild index 787d9842ea6..02528db1462 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-9999.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-9999.ebuild @@ -8,7 +8,7 @@ CROS_WORKON_REPO="https://github.com" if [[ "${PV}" == 9999 ]]; then KEYWORDS="~amd64 ~arm ~arm64 ~x86" else - CROS_WORKON_COMMIT="11309004b8451bf232539bdd3c7b0f5e4f69f5cb" # flatcar-master + CROS_WORKON_COMMIT="e4b55716dc08be6211026730f0bbf94e6ce44d51" # flatcar-master KEYWORDS="amd64 arm64" fi From 22582dca7ce92bb3e59fe5cc3f9fc0c7a9f2b215 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 19 Oct 2023 17:03:21 +0200 Subject: [PATCH 32/40] changelog: add torcx removal, sysext migration Signed-off-by: Thilo Fromm --- changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md | 1 - changelog/changes/2023-10-17-update-engine-torcx-removal.md | 1 - changelog/changes/2023-10-19-torcx-removal.md | 4 ++++ 3 files changed, 4 insertions(+), 2 deletions(-) delete mode 100644 changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md delete mode 100644 changelog/changes/2023-10-17-update-engine-torcx-removal.md create mode 100644 changelog/changes/2023-10-19-torcx-removal.md diff --git a/changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md b/changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md deleted file mode 100644 index 41f093195a9..00000000000 --- a/changelog/changes/2023-10-17-sdk-mantle-kola-sysext.md +++ /dev/null @@ -1 +0,0 @@ -- SDK: bump Mantle / Kola container image version to include torcx removal / sysext tests ([Mantle#466](https://github.com/flatcar/mantle/pull/466), [Mantle#465](https://github.com/flatcar/mantle/pull/465)) diff --git a/changelog/changes/2023-10-17-update-engine-torcx-removal.md b/changelog/changes/2023-10-17-update-engine-torcx-removal.md deleted file mode 100644 index 3e04769cec3..00000000000 --- a/changelog/changes/2023-10-17-update-engine-torcx-removal.md +++ /dev/null @@ -1 +0,0 @@ -- update_engine: remove torcx sanity checks because the OS image removed torcx ([update_engine#30](https://github.com/flatcar/update_engine/pull/30)) diff --git a/changelog/changes/2023-10-19-torcx-removal.md b/changelog/changes/2023-10-19-torcx-removal.md new file mode 100644 index 00000000000..fbf2339cb7a --- /dev/null +++ b/changelog/changes/2023-10-19-torcx-removal.md @@ -0,0 +1,4 @@ +- **torcx was replaced by systemd-sysext in the OS image**. Learn more about sysext and how to customise OS images [here](https://www.flatcar.org/docs/latest/provisioning/sysext/). + - Torcx has been removed entirely; if you use torcx to extend the Flatcar base OS image, please refer to our [conversion script](https://www.flatcar.org/docs/latest/provisioning/sysext/#torcx-deprecation) and to the sysext documentation mentioned above for migrating. + - Consequently, `update_engine` will not perform torcx sanity checks post-update anymore. + - Relevant changes: [scripts#1216](https://github.com/flatcar/scripts/pull/1216), [update_engine#30](https://github.com/flatcar/update_engine/pull/30), [Mantle#466](https://github.com/flatcar/mantle/pull/466), [Mantle#465](https://github.com/flatcar/mantle/pull/465). From 57ed77ea91bba4bc98b8e10d34ac50bc1ac70845 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 19 Oct 2023 18:38:06 +0200 Subject: [PATCH 33/40] ci-automation: fix local image detection for nightly SDKs Signed-off-by: Thilo Fromm --- ci-automation/ci_automation_common.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index ac71b04ffe4..283ea7f554e 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -192,9 +192,16 @@ function docker_image_from_buildcache() { local id_file_url="https://${BUILDCACHE_SERVER}/containers/${version}/${id_file}" local id_file_url_release="https://mirror.release.flatcar-linux.net/containers/${version}/${id_file}" - if image_exists_locally "${name}" "${version}" ; then + local local_image="" + if image_exists_locally "/${name}" "${version}" ; then + local_image="${name}:${version}" + elif image_exists_locally "${CONTAINER_REGISTRY}/${name}" "${version}" ; then + local_image="${CONTAINER_REGISTRY}/${name}:${version}" + fi + + if [[ -n "${local_image}" ]] ; then local image_id="" - image_id=$($docker image inspect "${name}:${version}" | jq -r '.[].Id' | sed 's/^sha256://') + image_id=$($docker image inspect "${local_image}" | jq -r '.[].Id' | sed 's/^sha256://') local remote_id="" remote_id=$(curl --fail --silent --show-error --location --retry-delay 1 \ --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 \ From b775036a628a06dc91fa04bcbe5a52fe3207be37 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 20 Oct 2023 16:37:30 +0200 Subject: [PATCH 34/40] torcx removal: address PR comments - updated github actions for runc, containerd, and docker to not handle nonexistent ebuilds in app-torcx/ anymore - removed spurious package_run_dependencies from build_image_util.sh - build_sysext: generate pkginfo before mangle script runs use zstd for compression; add cli flag to select compression - ci_automation_common.sh: remove spurious `/` from match string - coreos, board-packages, bootengine: bump ebuild revisions - kernel commonconfig: add squashfs zstd support Signed-off-by: Thilo Fromm --- .github/workflows/containerd-apply-patch.sh | 10 +--------- .github/workflows/docker-apply-patch.sh | 7 ------- .github/workflows/runc-apply-patch.sh | 8 +------- build_library/build_image_util.sh | 11 ---------- build_sysext | 20 ++++++++++--------- ci-automation/ci_automation_common.sh | 2 +- ...1-r309.ebuild => coreos-0.0.1-r310.ebuild} | 0 ...ebuild => board-packages-0.0.1-r12.ebuild} | 0 ...23.ebuild => bootengine-0.0.38-r24.ebuild} | 0 .../coreos-modules/files/commonconfig-6.1 | 1 + 10 files changed, 15 insertions(+), 44 deletions(-) rename sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/{coreos-0.0.1-r309.ebuild => coreos-0.0.1-r310.ebuild} (100%) rename sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/{board-packages-0.0.1-r11.ebuild => board-packages-0.0.1-r12.ebuild} (100%) rename sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/{bootengine-0.0.38-r23.ebuild => bootengine-0.0.38-r24.ebuild} (100%) diff --git a/.github/workflows/containerd-apply-patch.sh b/.github/workflows/containerd-apply-patch.sh index 12c02f7a1d8..27c706196bf 100755 --- a/.github/workflows/containerd-apply-patch.sh +++ b/.github/workflows/containerd-apply-patch.sh @@ -28,21 +28,13 @@ git mv "${containerdEbuildOldSymlink}" "${containerdEbuildNewSymlink}" sed -i "s/CONTAINERD_COMMIT=\"\(.*\)\"/CONTAINERD_COMMIT=\"${COMMIT_HASH}\"/g" "${containerdEbuildMain}" sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${containerdEbuildMain}" - -DOCKER_VERSION=$(sed -n "s/^DIST docker-\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1) -# torcx ebuild file has a docker version with only major and minor versions, like 19.03. -versionTorcx=${DOCKER_VERSION%.*} -torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}") -sed -i "s/containerd-${VERSION_OLD}/containerd-${VERSION_NEW}/g" "${torcxEbuildFile}" - popd URL="https://github.com/containerd/containerd/releases/tag/v${VERSION_NEW}" generate_update_changelog 'containerd' "${VERSION_NEW}" "${URL}" 'containerd' -commit_changes app-containers/containerd "${VERSION_OLD}" "${VERSION_NEW}" \ - app-torcx/docker +commit_changes app-containers/containerd "${VERSION_OLD}" "${VERSION_NEW}" cleanup_repo diff --git a/.github/workflows/docker-apply-patch.sh b/.github/workflows/docker-apply-patch.sh index 3c5a046386f..ac9606c45b2 100755 --- a/.github/workflows/docker-apply-patch.sh +++ b/.github/workflows/docker-apply-patch.sh @@ -33,12 +33,6 @@ git mv "${cliEbuildOld}" "${cliEbuildNew}" sed -i "s/GIT_COMMIT=\(.*\)/GIT_COMMIT=${COMMIT_HASH_CLI}/g" "${cliEbuildNew}" sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${cliEbuildNew}" -# torcx ebuild file has a docker version with only major and minor versions, like 19.03. -versionTorcx=${VERSION_OLD%.*} -torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}") -sed -i "s/docker-${VERSION_OLD}/docker-${VERSION_NEW}/g" "${torcxEbuildFile}" -sed -i "s/docker-cli-${VERSION_OLD}/docker-cli-${VERSION_NEW}/g" "${torcxEbuildFile}" - # update also docker versions used by the current runc ebuild file. versionRunc=$(sed -n "s/^DIST runc-\([0-9]*.[0-9]*.*\)\.tar.*/\1/p" app-containers/runc/Manifest | sort -ruV | head -n1) runcEbuildFile=$(get_ebuild_filename app-containers/runc "${versionRunc}") @@ -63,7 +57,6 @@ generate_update_changelog 'Docker' "${VERSION_NEW}" "${URL}" 'docker' regenerate_manifest app-containers/docker-cli "${VERSION_NEW}" commit_changes app-containers/docker "${VERSION_OLD}" "${VERSION_NEW}" \ app-containers/docker-cli \ - app-torcx/docker \ app-containers/runc cleanup_repo diff --git a/.github/workflows/runc-apply-patch.sh b/.github/workflows/runc-apply-patch.sh index 155132198c8..8aa1b4a4957 100755 --- a/.github/workflows/runc-apply-patch.sh +++ b/.github/workflows/runc-apply-patch.sh @@ -36,11 +36,6 @@ sed -i "s/runc-${VERSION_OLD}/runc-${VERSION_NEW}/g" app-containers/containerd/c dockerVersion=$(sed -n "s/^DIST docker-\([0-9]*.[0-9]*.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1) -# torcx ebuild file has a docker version with only major and minor versions, like 19.03. -versionTorcx=${dockerVersion%.*} -torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}") -sed -i "s/runc-${VERSION_OLD}/runc-${VERSION_NEW}/g" "${torcxEbuildFile}" - popd URL="https://github.com/opencontainers/runc/releases/tag/v${VERSION_NEW}" @@ -48,8 +43,7 @@ URL="https://github.com/opencontainers/runc/releases/tag/v${VERSION_NEW}" generate_update_changelog 'runc' "${VERSION_NEW}" "${URL}" 'runc' commit_changes app-containers/runc "${VERSION_OLD}" "${VERSION_NEW}" \ - app-containers/containerd \ - app-torcx/docker + app-containers/containerd cleanup_repo diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 790c1a9556c..95714c373a2 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -261,17 +261,6 @@ image_packages_portage() { equery --no-color list --format '$cpv::$repo' '*' } -# List dependencies for a package runtime dependencies - -function package_run_dependencies() ( - pkg=${1:?} - ebuild=$(equery-${BOARD} w "${pkg}") - function inherit() { : ; } - . "${ebuild}" - echo ${RDEPEND} -) - - # List packages implicitly contained in rootfs, such as in initramfs. image_packages_implicit() { local profile="${BUILD_DIR}/configroot/etc/portage/profile" diff --git a/build_sysext b/build_sysext index 95b00d49ce4..fa862727c13 100755 --- a/build_sysext +++ b/build_sysext @@ -31,6 +31,8 @@ DEFINE_boolean generate_pkginfo "${FLAGS_FALSE}" \ "Generate an additional squashfs '_pkginfo.raw' with portage package meta-information (/var/db ...). Useful for creating sysext dependencies; see 'base_pkginfo' below." DEFINE_string base_pkginfo "" \ "Colon-separated list of pkginfo squashfs paths / files generated via 'generate_pkginfo' to base this sysext on. The corresponding base sysexts are expected to be merged with the sysext generated." +DEFINE_string compression "zstd" \ + "Compression to use for sysext squashfs. One of 'gzip', 'lzo', 'lz4', 'xz', or 'zstd'. Must be supported by the Flatcar squashfs kernel module in order for the sysext to work." DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \ "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS." @@ -193,7 +195,7 @@ if [[ ${#} -lt 1 ]]; then show_help_if_requested -h fi -info "Building '${SYSEXTNAME}' with (meta-)packages '${@}' in '${BUILD_DIR}'". +info "Building '${SYSEXTNAME}' squashfs with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression". for package; do echo "Installing package into sysext image: $package" @@ -211,6 +213,13 @@ done umount "${BUILD_DIR}/install-root" umount "${BUILD_DIR}/fs-root" +if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then + info " Creating pkginfo squashfs '${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw'" + mkdir -p "${BUILD_DIR}/img-pkginfo/var/db" + cp -R "${BUILD_DIR}/install-root/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/" + mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend -comp "${FLAGS_compression}" +fi + if [[ -n "${FLAGS_manglefs_script}" ]]; then if [[ ! -x "${FLAGS_manglefs_script}" ]]; then die "${FLAGS_manglefs_script} is not executable" @@ -218,13 +227,6 @@ if [[ -n "${FLAGS_manglefs_script}" ]]; then "${FLAGS_manglefs_script}" "${BUILD_DIR}/install-root" fi -if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then - info " Creating pkginfo squashfs '${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw'" - mkdir -p "${BUILD_DIR}/img-pkginfo/var/db" - cp -R "${BUILD_DIR}/install-root/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/" - mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend -fi - info "Writing ${SYSEXTNAME}_packages.txt" ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root"\ equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" @@ -245,7 +247,7 @@ all_fields=( "ARCHITECTURE=${ARCH}" ) printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/install-root/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}" -mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" -noappend +mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" -noappend -comp "${FLAGS_compression}" rm -rf "${BUILD_DIR}"/{fs-root,install-root,workdir} # Generate reports diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index 283ea7f554e..a0745b25ecd 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -193,7 +193,7 @@ function docker_image_from_buildcache() { local id_file_url_release="https://mirror.release.flatcar-linux.net/containers/${version}/${id_file}" local local_image="" - if image_exists_locally "/${name}" "${version}" ; then + if image_exists_locally "${name}" "${version}" ; then local_image="${name}:${version}" elif image_exists_locally "${CONTAINER_REGISTRY}/${name}" "${version}" ; then local_image="${CONTAINER_REGISTRY}/${name}:${version}" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1-r309.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1-r310.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1-r309.ebuild rename to sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1-r310.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r11.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r12.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r11.ebuild rename to sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r12.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r23.ebuild b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r24.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r23.ebuild rename to sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r24.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/files/commonconfig-6.1 b/sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/files/commonconfig-6.1 index 24c15dc8a48..9980f79cbce 100644 --- a/sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/files/commonconfig-6.1 +++ b/sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/files/commonconfig-6.1 @@ -854,6 +854,7 @@ CONFIG_SQUASHFS_LZ4=y CONFIG_SQUASHFS_LZO=y CONFIG_SQUASHFS_XATTR=y CONFIG_SQUASHFS_XZ=y +CONFIG_SQUASHFS_ZSTD=y CONFIG_STACKPROTECTOR_STRONG=y CONFIG_STRICT_DEVMEM=y CONFIG_STRIP_ASM_SYMS=y From c2d1c13abdb51b9fb347e83e103fc2fef7f224f8 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 20 Oct 2023 16:41:13 +0200 Subject: [PATCH 35/40] Apply bash scripting suggestions from code review Thank you Krzesimir! Co-authored-by: Krzesimir Nowak --- build_library/sysext_prod_builder | 9 ++++----- run_local_tests.sh | 12 ++++++------ run_sdk_container | 6 +++--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/build_library/sysext_prod_builder b/build_library/sysext_prod_builder index a1db9acd7d1..73afbb6cb49 100755 --- a/build_library/sysext_prod_builder +++ b/build_library/sysext_prod_builder @@ -33,8 +33,7 @@ create_prod_sysext() { local grp_pkg="$7" local pkginfo="${8:-}" - local pkginfo_opt="" - local manglefs_opt="" + local -a build_sysext_opts=() local msg="Installing ${grp_pkg}' in sysext ${name}.raw" @@ -44,12 +43,12 @@ create_prod_sysext() { die "Sysext build '${grp_pkg}': unable to find package info at '${output_dir}/${pkginfo}'." fi msg="${msg} w/ package info '${pkginfo}'" - pkginfo_opt="--base_pkginfo=${output_dir}/${pkginfo}" + build_sysext_opts+=( "--base_pkginfo=${output_dir}/${pkginfo}" ) fi # Include FS mangle script if present if [[ -x "${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ]] ; then - manglefs_opt="--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" + build_sysext_opts+=( "--manglefs_script=${BUILD_LIBRARY_DIR}/sysext_mangle_${name}" ) msg="${msg}, FS mangle script 'sysext_mangle_${name}'" fi @@ -60,7 +59,7 @@ create_prod_sysext() { --image_builddir="${workdir}/sysext-build" \ --squashfs_base="${base_sysext}" \ --generate_pkginfo \ - ${manglefs_opt} ${pkginfo_opt} \ + "${build_sysext_opts[@]}" \ "${name}" "${grp_pkg}" sudo mv "${workdir}/sysext-build/${name}.raw" "${workdir}/sysext-build/${name}_pkginfo.raw" \ diff --git a/run_local_tests.sh b/run_local_tests.sh index f13c2e6bff7..046df79d5f1 100755 --- a/run_local_tests.sh +++ b/run_local_tests.sh @@ -51,10 +51,10 @@ function set_vars() { # Read by the mantle container. # The local directory ("pwd") will be mounted to /work/ in the container. cat > sdk_container/.env < Date: Fri, 20 Oct 2023 17:54:21 +0200 Subject: [PATCH 36/40] coreos/base/package.use: squashfs-tools: enable all compression formats This change enables USE flags for all supported compression formats. zstd specifically is required to build zstd sysexts. --- .../coreos-overlay/profiles/coreos/base/package.use | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use index 137a9e81afc..75a02185080 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use @@ -44,8 +44,8 @@ sys-boot/grub -multislot -nls -themes -fonts # disable "high performance ssh" patch, turn on kerberos net-misc/openssh -hpn kerberos -# xz and lzo are required to run grub tests -sys-fs/squashfs-tools lzo xz +# xz and lzo are required to run grub tests, zstd is the default for sysexts +sys-fs/squashfs-tools xz lz4 lzma lzo zstd # make the kernel version discoverable via the traditional gentoo symlink sys-kernel/coreos-sources symlink From d744d32733838fcdf6d76931b898561204098ddf Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 20 Oct 2023 18:41:41 +0200 Subject: [PATCH 37/40] build_sysext: optimise squashfs for size Signed-off-by: Thilo Fromm --- build_sysext | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/build_sysext b/build_sysext index fa862727c13..9b0b948c7ff 100755 --- a/build_sysext +++ b/build_sysext @@ -33,6 +33,8 @@ DEFINE_string base_pkginfo "" \ "Colon-separated list of pkginfo squashfs paths / files generated via 'generate_pkginfo' to base this sysext on. The corresponding base sysexts are expected to be merged with the sysext generated." DEFINE_string compression "zstd" \ "Compression to use for sysext squashfs. One of 'gzip', 'lzo', 'lz4', 'xz', or 'zstd'. Must be supported by the Flatcar squashfs kernel module in order for the sysext to work." +DEFINE_string mksquashfs_opts "" \ + "Additional command line options to pass to mksquashfs. See 'man 1 mksquashfs'. If is 'zstd' (the default), this option defaults to '-Xcompression-level 22 -b 512K'. Otherwise the default is empty." DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \ "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS." @@ -100,6 +102,10 @@ fi BUILD_DIR=$(realpath "${FLAGS_image_builddir}") mkdir -p "${BUILD_DIR}" +if [[ "${FLAGS_compression}" = "zstd" && -z "${FLAGS_mksquashfs_opts}" ]] ; then + FLAGS_mksquashfs_opts="-Xcompression-level 22 -b 512k" +fi + source "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 @@ -217,7 +223,8 @@ if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then info " Creating pkginfo squashfs '${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw'" mkdir -p "${BUILD_DIR}/img-pkginfo/var/db" cp -R "${BUILD_DIR}/install-root/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/" - mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend -comp "${FLAGS_compression}" + mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" \ + -noappend -comp "${FLAGS_compression}" ${FLAGS_mksquashfs_opts} fi if [[ -n "${FLAGS_manglefs_script}" ]]; then @@ -247,7 +254,8 @@ all_fields=( "ARCHITECTURE=${ARCH}" ) printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/install-root/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}" -mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" -noappend -comp "${FLAGS_compression}" +mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" \ + -noappend -comp "${FLAGS_compression}" ${FLAGS_mksquashfs_opts} rm -rf "${BUILD_DIR}"/{fs-root,install-root,workdir} # Generate reports From 6dd2f203f1a4bc2a4477d1dda59534def2b5836b Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 20 Oct 2023 18:51:33 +0200 Subject: [PATCH 38/40] run_local_tests.sh: un-break variable quoting Turns out using ${var@Q} instead of ${var} ends up with paths like /work/foo/'amd64'-usr/... instead of /work/foo/amd64-usr/... which breaks the script. So we revert it. Signed-off-by: Thilo Fromm --- run_local_tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run_local_tests.sh b/run_local_tests.sh index 046df79d5f1..a3c67ae4cc3 100755 --- a/run_local_tests.sh +++ b/run_local_tests.sh @@ -51,10 +51,10 @@ function set_vars() { # Read by the mantle container. # The local directory ("pwd") will be mounted to /work/ in the container. cat > sdk_container/.env < Date: Mon, 23 Oct 2023 16:03:21 +0200 Subject: [PATCH 39/40] Update run_local_tests.sh: improve quoting Co-authored-by: Krzesimir Nowak --- run_local_tests.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run_local_tests.sh b/run_local_tests.sh index a3c67ae4cc3..e70676a3776 100755 --- a/run_local_tests.sh +++ b/run_local_tests.sh @@ -51,10 +51,10 @@ function set_vars() { # Read by the mantle container. # The local directory ("pwd") will be mounted to /work/ in the container. cat > sdk_container/.env < Date: Mon, 23 Oct 2023 16:04:53 +0200 Subject: [PATCH 40/40] build_sysext: run package inventorisation before mangle script Signed-off-by: Thilo Fromm --- build_sysext | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build_sysext b/build_sysext index 9b0b948c7ff..bd819e281cf 100755 --- a/build_sysext +++ b/build_sysext @@ -227,6 +227,10 @@ if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then -noappend -comp "${FLAGS_compression}" ${FLAGS_mksquashfs_opts} fi +info "Writing ${SYSEXTNAME}_packages.txt" +ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root" \ + equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" + if [[ -n "${FLAGS_manglefs_script}" ]]; then if [[ ! -x "${FLAGS_manglefs_script}" ]]; then die "${FLAGS_manglefs_script} is not executable" @@ -234,10 +238,6 @@ if [[ -n "${FLAGS_manglefs_script}" ]]; then "${FLAGS_manglefs_script}" "${BUILD_DIR}/install-root" fi -info "Writing ${SYSEXTNAME}_packages.txt" -ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root"\ - equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt" - info "Removing non-/usr directories from sysext image" for entry in "${BUILD_DIR}/install-root"/*; do if [[ "${entry}" = */usr ]]; then