Merge pull request #1016 from flatcar/kai/update-sysext

Support OEM systemd-sysext images and Flatcar extensions
flatcar · Sep 8, 2023 · 6bec600 · 6bec600
2 parents 096b51b + ddb808f
commit 6bec600
Show file tree

Hide file tree

Showing 13 changed files with 60 additions and 12 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -120,6 +120,7 @@ jobs:
           set -x
           set -euo pipefail
 
+          # This is also done again in run-kola-tests.yaml because these changes here disappear
           source ci-automation/ci_automation_common.sh
           source sdk_container/.repo/manifests/version.txt
 

diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml
@@ -205,6 +205,20 @@ jobs:
           python3 -m http.server -d "${TESTS_WEBSERVER_WEBROOT}" -b "${TESTS_WEBSERVER_IP}" "${TESTS_WEBSERVER_PORT}" &
 
           pushd scripts
+
+          source ci-automation/ci_automation_common.sh
+          source sdk_container/.repo/manifests/version.txt
+
+          version="alpha-$FLATCAR_VERSION_ID"
+          check_version_string "$version"
+          sdk_version="${CUSTOM_SDK_VERSION:-$FLATCAR_SDK_VERSION}"
+
+          # Create version file
+          (
+            source sdk_lib/sdk_container_common.sh
+            create_versionfile "$sdk_version" "$version"
+          )
+
           source ci-automation/test.sh
 
           # Provide our own torcx prepare function so we use our local manifest json.

diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh
@@ -531,12 +531,7 @@ install_oem_sysext() {
     local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext"
     local built_sysext_filename="${oem_sysext}.raw"
     local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}"
-    # TODO: Set 'version' to "${FLATCAR_VERSION}" and drop
-    # VERSION_FIELD_OVERRIDE when we implement updating OEM sysexts.
-    local version='initial'
-    local build_sysext_env=(
-        VERSION_FIELD_OVERRIDE='SYSEXT_LEVEL=1.0'
-    )
+    local version='${FLATCAR_VERSION}'
     local metapkg="coreos-base/${oem_sysext}"
     local build_sysext_flags=(
         --board="${BOARD}"

diff --git a/changelog/changes/2023-09-06-oem-vendor-tool-updates.md b/changelog/changes/2023-09-06-oem-vendor-tool-updates.md
@@ -0,0 +1 @@
+- OEM vendor tools are now A/B updated if they are shipped as systemd-sysext images, the migration happens when both partitions require a systemd-sysext OEM image - note that this will delete the `nvidia.service` from `/etc` on Azure because it's now part of `/usr` ([Flatcar#60](https://github.com/flatcar/Flatcar/issues/60))
diff --git a/ci-automation/vendor-testing/qemu_update.sh b/ci-automation/vendor-testing/qemu_update.sh
@@ -13,11 +13,11 @@ set -euo pipefail
 source ci-automation/vendor_test.sh
 
 # The last check is not perfect (if both tests are rerun, it will only look at the name of the second test) but hopefully still good enough to prevent wrong usage
-if [ "$*" != "" ] && [ "$*" != "*" ] && [[ "$*" != *"cl.update.payload" ]]; then
+if [ "$*" != "" ] && [ "$*" != "*" ] && [[ "$*" != *"cl.update."* ]]; then
     echo "1..1" > "${CIA_TAPFILE}"
     echo "not ok - all qemu update tests" >> "${CIA_TAPFILE}"
     echo "  ---" >> "${CIA_TAPFILE}"
-    echo "  ERROR: Only cl.update.payload is supported, got '$*'." | tee -a "${CIA_TAPFILE}"
+    echo "  ERROR: Only cl.update.payload and cl.update.oem are supported, got '$*'." | tee -a "${CIA_TAPFILE}"
     echo "  ..." >> "${CIA_TAPFILE}"
     break_retest_cycle
     exit 1
@@ -27,6 +27,12 @@ mkdir -p "$(dirname ${QEMU_UPDATE_PAYLOAD})"
 if [ -f "${QEMU_UPDATE_PAYLOAD}" ] ; then
     echo "++++ ${CIA_TESTSCRIPT}: Using existing ${QEMU_UPDATE_PAYLOAD} for testing ${CIA_VERNUM} (${CIA_ARCH}) ++++"
 else
+    # TODO: Change the GitHub Action to provide this artifact and detect that case here and skip the bincache download
+    if ! curl --head -o /dev/null -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://bincache.flatcar-linux.net/images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update.gz"; then
+      echo "1..1" > "${CIA_TAPFILE}"
+      echo "ok - skipped qemu update tests" >> "${CIA_TAPFILE}"
+      exit 0
+    fi
     echo "++++ ${CIA_TESTSCRIPT}: downloading flatcar_test_update.gz for ${CIA_VERNUM} (${CIA_ARCH}) ++++"
     copy_from_buildcache "images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update.gz" tmp/
 fi
@@ -87,11 +93,16 @@ query_kola_tests() {
 run_kola_tests() {
     local instance_type="${1}"; shift;
     local instance_tapfile="${1}"; shift
+    local tests=("cl.update.payload")
     local image
     if [ "${instance_type}" = "previous" ]; then
         image="tmp/flatcar_production_image_previous.bin"
     elif [ "${instance_type}" = "first_dual" ]; then
         image="tmp/flatcar_production_image_first_dual.bin"
+        # Only run this test if the Azure dev payload exists on bincache because the fallback download needs it
+        if curl --head -o /dev/null -fsSL --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://bincache.flatcar-linux.net/images/${CIA_ARCH}/${CIA_VERNUM}/flatcar_test_update-oem-azure.gz"; then
+          tests+=("cl.update.oem")
+        fi
     else
         echo "Wrong instance type ${instance_type}" >&2
         exit 1
@@ -107,7 +118,7 @@ run_kola_tests() {
       --torcx-manifest="${CIA_TORCX_MANIFEST}" \
       --update-payload="${QEMU_UPDATE_PAYLOAD}" \
       ${QEMU_KOLA_SKIP_MANGLE:+--qemu-skip-mangle} \
-      cl.update.payload
+      "${tests[@]}"
 }
 
 run_kola_tests_on_instances "previous" "${CIA_TAPFILE}" "${CIA_FIRST_RUN}" first_dual -- cl.update.payload -- "${@}"
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild
@@ -10,7 +10,7 @@ CROS_WORKON_REPO="https://github.com"
 if [[ "${PV}" == 9999 ]]; then
 	KEYWORDS="~amd64 ~arm ~arm64 ~x86"
 else
-	CROS_WORKON_COMMIT="235b6141a0b6eb4c1e4dc7778a896f22756898a3" # flatcar-master
+	CROS_WORKON_COMMIT="a7c8b0c7ba4209e520532d1b92131e8ad23c40fb" # flatcar-master
 	KEYWORDS="amd64 arm arm64 x86"
 fi
 

diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/files/oems/azure b/sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/files/oems/azure
@@ -0,0 +1,13 @@
+/etc/systemd/system/nvidia.service
+/oem/bin/setup-nvidia
+/oem/bin/install-nvidia
+/oem/units/nvidia.service
+/etc/systemd/system/oem-cloudinit.service
+/etc/systemd/system/multi-user.target.wants/oem-cloudinit.service
+/etc/systemd/system/waagent.service
+/etc/systemd/system/multi-user.target.wants/waagent.service
+/oem/waagent.conf
+/oem/python/
+/oem/bin/
+/oem/units/
+/oem/base/
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/files/oems/qemu b/sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/files/oems/qemu
diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/misc-files-0.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/misc-files/misc-files-0.ebuild
@@ -79,6 +79,14 @@ src_install() {
     insinto '/etc/bash/bashrc.d'
     doins "${FILESDIR}/99-flatcar-bcc"
 
+    insinto '/usr/share/flatcar'
+    # The "oems" folder should contain a file "$OEMID" for each expected OEM sysext and
+    # either be empty or contain a newline-separated list of files to delete during the
+    # migration (done from the initrd). The existence of the file will help old clients
+    # to do the fallback download of the sysext payload in the postinstall hook.
+    # The paths should use /oem instead of /usr/share/oem/ to avoid symlink resolution.
+    doins -r "${FILESDIR}"/oems
+
     dotmpfiles "${T}/home-core-bash-symlinks.conf"
     # Ideally we would be calling systemd-tmpfiles to create the
     # symlinks, but at this point systemd may not have any info about

diff --git a/...te_engine/update_engine-0.4.10-r10.ebuild → ...te_engine/update_engine-0.4.10-r11.ebuild b/...te_engine/update_engine-0.4.10-r10.ebuild → ...te_engine/update_engine-0.4.10-r11.ebuild
diff --git a/...tainer/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-9999.ebuild b/...tainer/src/third_party/coreos-overlay/coreos-base/update_engine/update_engine-9999.ebuild
@@ -8,7 +8,7 @@ CROS_WORKON_REPO="https://github.com"
 if [[ "${PV}" == 9999 ]]; then
 	KEYWORDS="~amd64 ~arm ~arm64 ~x86"
 else
-	CROS_WORKON_COMMIT="c6f566d47d8949632f7f43871eb8d5c625af3209" # flatcar-master
+	CROS_WORKON_COMMIT="b3cdd63f96cfa35c73c56dc6ce4fbe6cbc5d6740" # flatcar-master
 	KEYWORDS="amd64 arm64"
 fi
 
@@ -102,6 +102,11 @@ src_install() {
 	insinto /usr/share/dbus-1/system.d
 	doins com.coreos.update1.conf
 
+	insinto /usr/share/update_engine
+	doins src/update_engine/update_metadata.proto
+	exeinto /usr/share/update_engine
+	doexe decode_payload
+
 	# Install rule to remove old UpdateEngine.conf from /etc
 	dotmpfiles "${FILESDIR}"/update-engine.conf
 }
diff --git a/...l/bootengine/bootengine-0.0.38-r17.ebuild → ...l/bootengine/bootengine-0.0.38-r18.ebuild b/...l/bootengine/bootengine-0.0.38-r17.ebuild → ...l/bootengine/bootengine-0.0.38-r18.ebuild
diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild
@@ -10,7 +10,7 @@ CROS_WORKON_REPO="https://github.com"
 if [[ "${PV}" == 9999 ]]; then
 	KEYWORDS="~amd64 ~arm ~arm64 ~x86"
 else
-	CROS_WORKON_COMMIT="cc0fdec0cc6c5692acac95a928984ea8a5eb8f08" # flatcar-master
+	CROS_WORKON_COMMIT="1066c7f5155f9846da3f83473be9c8768524b52c" # flatcar-master
 	KEYWORDS="amd64 arm arm64 x86"
 fi