Add the infrastructure for shim signing & aarch64 support

.github/workflows/ci.yaml

@@ -278,7 +278,7 @@ jobs:
             scripts/artifacts/images/flatcar_production_image*.txt
             scripts/artifacts/images/flatcar_production_image*.json
             scripts/artifacts/images/flatcar_production_image_pcr_policy.zip
-            scripts/artifacts/images/flatcar_production_*_efi_*.fd
+            scripts/artifacts/images/flatcar_production_*_efi_*.qcow2
             scripts/artifacts/images/flatcar_production_qemu.sh
 
       - name: Upload developer container
@@ -317,7 +317,7 @@ jobs:
           path: |
             scripts/artifacts/images/*.img
             scripts/artifacts/images/*.bin
-            scripts/artifacts/images/flatcar_production_*_efi_*.fd
+            scripts/artifacts/images/flatcar_production_*_efi_*.qcow2
             scripts/artifacts/images/*.txt
             scripts/artifacts/images/flatcar-*.raw
             scripts/artifacts/images/flatcar_production_*.sh

.github/workflows/portage-stable-packages-list

@@ -117,13 +117,13 @@ app-containers/runc
 app-crypt/adcli
 app-crypt/argon2
 app-crypt/ccid
-app-crypt/efitools
 app-crypt/gnupg
 app-crypt/gpgme
 app-crypt/libb2
 app-crypt/libmd
 app-crypt/mhash
 app-crypt/mit-krb5
+app-crypt/p11-kit
 app-crypt/pinentry
 app-crypt/rhash
 app-crypt/shash
@@ -138,6 +138,7 @@ app-editors/vim-core
 
 app-emulation/qemu
 app-emulation/qemu-guest-agent
+app-emulation/virt-firmware
 
 app-eselect/eselect-iptables
 app-eselect/eselect-lib-bin-symlink
@@ -188,6 +189,10 @@ dev-build/meson-format-array
 dev-build/ninja
 
 dev-cpp/abseil-cpp
+dev-cpp/azure-core
+dev-cpp/azure-identity
+dev-cpp/azure-security-keyvault-certificates
+dev-cpp/azure-security-keyvault-keys
 dev-cpp/gflags
 dev-cpp/glog
 dev-cpp/gtest
@@ -284,10 +289,12 @@ dev-python/autocommand
 dev-python/backports-tarfile
 dev-python/cachecontrol
 dev-python/certifi
+dev-python/cffi
 dev-python/chardet
 dev-python/charset-normalizer
 dev-python/colorama
 dev-python/crcmod
+dev-python/cryptography
 dev-python/cython
 dev-python/distlib
 dev-python/distro
@@ -322,11 +329,14 @@ dev-python/olefile
 dev-python/ordered-set
 dev-python/packaging
 dev-python/pathspec
+dev-python/pefile
 dev-python/pillow
 dev-python/pip
 dev-python/platformdirs
 dev-python/pluggy
+dev-python/ply
 dev-python/poetry-core
+dev-python/pycparser
 dev-python/pydecomp
 dev-python/pygments
 dev-python/pyproject-hooks
@@ -355,6 +365,7 @@ dev-util/desktop-file-utils
 dev-util/gdbus-codegen
 dev-util/glib-utils
 dev-util/gperf
+dev-util/maturin
 dev-util/pahole
 dev-util/patchelf
 dev-util/patchutils
@@ -450,6 +461,7 @@ eclass/savedconfig.eclass
 eclass/secureboot.eclass
 eclass/selinux-policy-2.eclass
 eclass/sgml-catalog-r1.eclass
+eclass/shell-completion.eclass
 eclass/ssl-cert.eclass
 eclass/strip-linguas.eclass
 eclass/subversion.eclass

.github/workflows/run-kola-tests.yaml

@@ -162,7 +162,7 @@ jobs:
           # Extract the generic image we'll use for qemu tests.
           # Note that the qemu[_uefi] tests use the generic image instead of the
           #  qemu vendor VM image ("Astronaut: [...] Always have been.").
-          mv flatcar_production_image.bin flatcar_production_qemu_uefi_efi_code.fd flatcar_production_qemu_uefi_efi_vars.fd scripts/
+          mv flatcar_production_image.bin flatcar_production_qemu_uefi_efi_code.qcow2 flatcar_production_qemu_uefi_efi_vars.qcow2 scripts/
 
           mv flatcar_test_update.gz scripts/
 
@@ -197,8 +197,8 @@ jobs:
           cat > sdk_container/.env <<EOF
           # export the QEMU_IMAGE_NAME to avoid to download it.
           export QEMU_IMAGE_NAME="/work/flatcar_production_image.bin"
-          export QEMU_UEFI_FIRMWARE="/work/flatcar_production_qemu_uefi_efi_code.fd"
-          export QEMU_UEFI_OVMF_VARS="/work/flatcar_production_qemu_uefi_efi_vars.fd"
+          export QEMU_UEFI_FIRMWARE="/work/flatcar_production_qemu_uefi_efi_code.qcow2"
+          export QEMU_UEFI_OVMF_VARS="/work/flatcar_production_qemu_uefi_efi_vars.qcow2"
           export QEMU_UPDATE_PAYLOAD="/work/flatcar_test_update.gz"
           export QEMU_DEVCONTAINER_URL="http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}"
           export QEMU_DEVCONTAINER_BINHOST_URL="http://${TESTS_WEBSERVER_IP}:${TESTS_WEBSERVER_PORT}"

build_library/build_image_util.sh

@@ -20,6 +20,7 @@ BUILD_DIR="${FLAGS_output_root}/${BOARD}/${IMAGE_SUBDIR}"
 OUTSIDE_OUTPUT_DIR="../build/images/${BOARD}/${IMAGE_SUBDIR}"
 
 source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1
+source "${BUILD_LIBRARY_DIR}/sbsign_util.sh" || exit 1
 
 set_build_symlinks() {
     local build=$(basename ${BUILD_DIR})
@@ -826,13 +827,8 @@ EOF
   fi
 
   # Sign the kernel after /usr is in a consistent state and verity is calculated
-  if [[ ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
-      sudo sbsign --key /usr/share/sb_keys/shim.key \
-	   --cert /usr/share/sb_keys/shim.pem \
-	   "${root_fs_dir}/boot/flatcar/vmlinuz-a"
-      sudo mv "${root_fs_dir}/boot/flatcar/vmlinuz-a.signed" \
-	   "${root_fs_dir}/boot/flatcar/vmlinuz-a"
-  fi
+  do_sbsign --output "${root_fs_dir}/boot/flatcar/vmlinuz-a"{,}
+  cleanup_sbsign_certs
 
   if [[ -n "${image_kernel}" ]]; then
     # copying kernel from vfat so ignore the permissions

build_library/grub_install.sh

@@ -35,56 +35,49 @@ switch_to_strict_mode
 # must be sourced after flags are parsed.
 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/sbsign_util.sh" || exit 1
 
 # Our GRUB lives under flatcar/grub so new pygrub versions cannot find grub.cfg
 GRUB_DIR="flatcar/grub/${FLAGS_target}"
 
-# GRUB install location inside the SDK
-GRUB_SRC="/usr/lib/grub/${FLAGS_target}"
-
 # Modules required to boot a standard CoreOS configuration
 CORE_MODULES=( normal search test fat part_gpt search_fs_uuid gzio search_part_label terminal gptprio configfile memdisk tar echo read btrfs )
 
-# Name of the core image, depends on target
-CORE_NAME=
-
-# Whether the SDK's grub or the board root's grub is used. Once amd64 is
-# fixed up the board root's grub will always be used.
-BOARD_GRUB=1
-
 SBAT_ARG=()
 
 case "${FLAGS_target}" in
-    i386-pc)
-        CORE_MODULES+=( biosdisk serial )
-        CORE_NAME="core.img"
-        ;;
     x86_64-efi)
-        CORE_MODULES+=( serial linux efi_gop efinet pgp http tftp tpm )
-        CORE_NAME="core.efi"
-        SBAT_ARG=( --sbat "${BOARD_ROOT}/usr/share/grub/sbat.csv" )
-        ;;
-    x86_64-xen)
-        CORE_NAME="core.elf"
+        EFI_ARCH="x64"
         ;;
     arm64-efi)
+        EFI_ARCH="aa64"
+        ;;
+esac
+
+case "${FLAGS_target}" in
+    x86_64-efi|arm64-efi)
+        GRUB_IMAGE="EFI/boot/grub${EFI_ARCH}.efi"
         CORE_MODULES+=( serial linux efi_gop efinet pgp http tftp tpm )
-        CORE_NAME="core.efi"
-        BOARD_GRUB=1
         SBAT_ARG=( --sbat "${BOARD_ROOT}/usr/share/grub/sbat.csv" )
         ;;
+    i386-pc)
+        GRUB_IMAGE="${GRUB_DIR}/core.img"
+        CORE_MODULES+=( biosdisk serial )
+        ;;
+    x86_64-xen)
+        GRUB_IMAGE="xen/pvboot-x86_64.elf"
+        ;;
     *)
         die_notrace "Unknown GRUB target ${FLAGS_target}"
         ;;
 esac
 
-if [[ $BOARD_GRUB -eq 1 ]]; then
-    info "Updating GRUB in ${BOARD_ROOT}"
-    emerge-${BOARD} \
-           --nodeps --select --verbose --update --getbinpkg --usepkgonly --newuse \
-           sys-boot/grub
-    GRUB_SRC="${BOARD_ROOT}/usr/lib/grub/${FLAGS_target}"
-fi
+info "Updating GRUB in ${BOARD_ROOT}"
+emerge-${BOARD} \
+        --nodeps --select --verbose --update --getbinpkg --usepkgonly --newuse \
+        sys-boot/grub
+
+GRUB_SRC="${BOARD_ROOT}/usr/lib/grub/${FLAGS_target}"
 [[ -d "${GRUB_SRC}" ]] || die "GRUB not installed at ${GRUB_SRC}"
 
 # In order for grub-setup-bios to properly detect the layout of the disk
@@ -97,6 +90,7 @@ ESP_DIR=
 LOOP_DEV=
 
 cleanup() {
+    cleanup_sbsign_certs
     if [[ -d "${ESP_DIR}" ]]; then
         if mountpoint -q "${ESP_DIR}"; then
             sudo umount "${ESP_DIR}"
@@ -130,7 +124,7 @@ done
 if [[ -z ${MOUNTED} ]]; then
     failboat "${LOOP_DEV}p1 where art thou? udev has forsaken us!"
 fi
-sudo mkdir -p "${ESP_DIR}/${GRUB_DIR}"
+sudo mkdir -p "${ESP_DIR}/${GRUB_DIR}" "${ESP_DIR}/${GRUB_IMAGE%/*}"
 
 info "Compressing modules in ${GRUB_DIR}"
 for file in "${GRUB_SRC}"/*{.lst,.mod}; do
@@ -172,15 +166,15 @@ if [[ ! -f "${ESP_DIR}/flatcar/grub/grub.cfg.tar" ]]; then
       -C "${GRUB_TEMP_DIR}" "grub.cfg"
 fi
 
-info "Generating ${GRUB_DIR}/${CORE_NAME}"
+info "Generating ${GRUB_IMAGE}"
 sudo grub-mkimage \
     --compression=auto \
     --format "${FLAGS_target}" \
     --directory "${GRUB_SRC}" \
     --config "${ESP_DIR}/${GRUB_DIR}/load.cfg" \
     --memdisk "${ESP_DIR}/flatcar/grub/grub.cfg.tar" \
     "${SBAT_ARG[@]}" \
-    --output "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \
+    --output "${ESP_DIR}/${GRUB_IMAGE}" \
     "${CORE_MODULES[@]}"
 
 for mod in "${CORE_MODULES[@]}"; do
@@ -189,77 +183,53 @@ done
 
 # Now target specific steps to make the system bootable
 case "${FLAGS_target}" in
-    i386-pc)
-        info "Installing MBR and the BIOS Boot partition."
-        sudo cp "${GRUB_SRC}/boot.img" "${ESP_DIR}/${GRUB_DIR}"
-        sudo grub-bios-setup --device-map=/dev/null \
-            --directory="${ESP_DIR}/${GRUB_DIR}" "${LOOP_DEV}"
-        # boot.img gets manipulated by grub-bios-setup so it alone isn't
-        # sufficient to restore the MBR boot code if it gets corrupted.
-        sudo dd bs=448 count=1 status=none if="${LOOP_DEV}" \
-            of="${ESP_DIR}/${GRUB_DIR}/mbr.bin"
-        ;;
-    x86_64-efi)
-        info "Installing default x86_64 UEFI bootloader."
-        sudo mkdir -p "${ESP_DIR}/EFI/boot"
-        # Use the test keys for signing unofficial builds
-        if [[ ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
-            # Sign the GRUB with the shim-embedded key
-            sudo sbsign --key /usr/share/sb_keys/shim.key \
-                --cert /usr/share/sb_keys/shim.pem \
-                "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}"
-            sudo mv "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}.signed" \
-                "${ESP_DIR}/EFI/boot/grubx64.efi"
-            sudo rm "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}"
-            # Sign the mokmanager(mm) with the shim-embedded key
-            sudo sbsign --key /usr/share/sb_keys/shim.key \
-                --cert /usr/share/sb_keys/shim.pem \
-                "/usr/lib/shim/mmx64.efi"
-            sudo cp "/usr/lib/shim/mmx64.efi.signed" \
-                "${ESP_DIR}/EFI/boot/mmx64.efi"
+    x86_64-efi|arm64-efi)
+        info "Installing default ${FLAGS_target} UEFI bootloader."
+
+        # Sign GRUB and mokmanager(mm) with the shim-embedded key.
+        do_sbsign --output "${ESP_DIR}/${GRUB_IMAGE}"{,}
+        do_sbsign --output "${ESP_DIR}/EFI/boot/mm${EFI_ARCH}.efi" \
+            "${BOARD_ROOT}/usr/lib/shim/mm${EFI_ARCH}.efi"
 
-            sudo sbsign --key /usr/share/sb_keys/DB.key \
+        if [[ ${COREOS_OFFICIAL:-0} -ne 1 ]]; then
+            # Unofficial build: Sign shim with our development key.
+            sudo sbsign \
+                --key /usr/share/sb_keys/DB.key \
                 --cert /usr/share/sb_keys/DB.crt \
-                --output "${ESP_DIR}/EFI/boot/bootx64.efi" \
-                "/usr/lib/shim/shim.efi"
+                --output "${ESP_DIR}/EFI/boot/boot${EFI_ARCH}.efi" \
+                "${BOARD_ROOT}/usr/lib/shim/shim${EFI_ARCH}.efi"
         else
-            sudo mv "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \
-                "${ESP_DIR}/EFI/boot/grubx64.efi"
-            sudo cp "/usr/lib/shim/shim.efi" \
-                "${ESP_DIR}/EFI/boot/bootx64.efi"
-            sudo cp "/usr/lib/shim/mmx64.efi" \
-                "${ESP_DIR}/EFI/boot/mmx64.efi"
+            # Official build: Copy our pre-signed shim.
+            sudo cp "${BOARD_ROOT}/usr/lib/shim/shim${EFI_ARCH}.efi.signed" \
+                "${ESP_DIR}/EFI/boot/boot${EFI_ARCH}.efi"
         fi
+
         # copying from vfat so ignore permissions
-        if [[ -n "${FLAGS_copy_efi_grub}" ]]; then
-            cp --no-preserve=mode "${ESP_DIR}/EFI/boot/grubx64.efi" \
+        if [[ -n ${FLAGS_copy_efi_grub} ]]; then
+            cp --no-preserve=mode "${ESP_DIR}/${GRUB_IMAGE}" \
                 "${FLAGS_copy_efi_grub}"
         fi
-        if [[ -n "${FLAGS_copy_shim}" ]]; then
-            cp --no-preserve=mode "${ESP_DIR}/EFI/boot/bootx64.efi" \
+        if [[ -n ${FLAGS_copy_shim} ]]; then
+            cp --no-preserve=mode "${ESP_DIR}/EFI/boot/boot${EFI_ARCH}.efi" \
                 "${FLAGS_copy_shim}"
         fi
         ;;
+    i386-pc)
+        info "Installing MBR and the BIOS Boot partition."
+        sudo cp "${GRUB_SRC}/boot.img" "${ESP_DIR}/${GRUB_DIR}"
+        sudo grub-bios-setup --device-map=/dev/null \
+            --directory="${ESP_DIR}/${GRUB_DIR}" "${LOOP_DEV}"
+        # boot.img gets manipulated by grub-bios-setup so it alone isn't
+        # sufficient to restore the MBR boot code if it gets corrupted.
+        sudo dd bs=448 count=1 status=none if="${LOOP_DEV}" \
+            of="${ESP_DIR}/${GRUB_DIR}/mbr.bin"
+        ;;
     x86_64-xen)
         info "Installing default x86_64 Xen bootloader."
-        sudo mkdir -p "${ESP_DIR}/xen" "${ESP_DIR}/boot/grub"
-        sudo mv "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \
-            "${ESP_DIR}/xen/pvboot-x86_64.elf"
+        sudo mkdir -p "${ESP_DIR}/boot/grub"
         sudo cp "${BUILD_LIBRARY_DIR}/menu.lst" \
             "${ESP_DIR}/boot/grub/menu.lst"
         ;;
-    arm64-efi)
-        info "Installing default arm64 UEFI bootloader."
-        sudo mkdir -p "${ESP_DIR}/EFI/boot"
-        #FIXME(andrejro): shim not ported to aarch64
-        sudo mv "${ESP_DIR}/${GRUB_DIR}/${CORE_NAME}" \
-            "${ESP_DIR}/EFI/boot/bootaa64.efi"
-        if [[ -n "${FLAGS_copy_efi_grub}" ]]; then
-            # copying from vfat so ignore permissions
-            cp --no-preserve=mode "${ESP_DIR}/EFI/boot/bootaa64.efi" \
-                "${FLAGS_copy_efi_grub}"
-        fi
-        ;;
 esac
 
 cleanup

build_library/qemu_template.sh

@@ -274,8 +274,8 @@ fi
 
 if [ -n "${VM_PFLASH_RO}" ] && [ -n "${VM_PFLASH_RW}" ]; then
     set -- \
-        -drive if=pflash,unit=0,file="${SCRIPT_DIR}/${VM_PFLASH_RO}",format=raw,readonly=on \
-        -drive if=pflash,unit=1,file="${SCRIPT_DIR}/${VM_PFLASH_RW}",format=raw "$@"
+        -drive if=pflash,unit=0,file="${SCRIPT_DIR}/${VM_PFLASH_RO}",format=qcow2,readonly=on \
+        -drive if=pflash,unit=1,file="${SCRIPT_DIR}/${VM_PFLASH_RW}",format=qcow2 "$@"
 fi
 
 if [ -n "${IGNITION_CONFIG_FILE}" ]; then