Implement raw disk image generation in go (#139)

* Implement raw disk image generation in go Drops the use of the script to generate it on pure golang Also drops some packages which are no longer needed from luet as we use the artifacts from the rootfs to support secureboot Signed-off-by: Itxaka <[email protected]> * Use source directly Aurora parses the source already and gives us a dir only Signed-off-by: Itxaka <[email protected]> * Pass only the output dir to the raw image generation Image name is generated formt he rootfs Signed-off-by: Itxaka <[email protected]> * Adress some comments - use a TempDir function to set/get the tmpdir - make all methods private - Fix the flavor/model get method to fallback to os-release Signed-off-by: Itxaka <[email protected]> * Use default constants for efi/oem partitions Signed-off-by: Itxaka <[email protected]> * Several improvements - fix gce and azure scripts, they were broken but alwasy exited with 0 so images generated were wrong (problems with parsing the image size) - improve the flavor/artifact name retrieval to be backwards compatible - simplify the call to gce/vhd transformation. It requires only the source dir where the raw image is and moves it witht he same name+new extension for easy identification - Fix and improve raw image tests. - More error catching and logging Signed-off-by: Itxaka <[email protected]> * Remove focus Signed-off-by: Itxaka <[email protected]> * Fix size not being parsed Signed-off-by: Itxaka <[email protected]> * Add tests for bootable raw image Signed-off-by: Itxaka <[email protected]> * Fix tests Signed-off-by: Itxaka <[email protected]> * Fix firmware path Signed-off-by: Itxaka <[email protected]> * Fix packages Signed-off-by: Itxaka <[email protected]> * test Signed-off-by: Itxaka <[email protected]> * Fix ovmf path Signed-off-by: Itxaka <[email protected]> * Fix Signed-off-by: Itxaka <[email protected]> * Run on kvm Signed-off-by: Itxaka <[email protected]> * Serial disk tests Signed-off-by: Itxaka <[email protected]> * Fix Signed-off-by: Itxaka <[email protected]> * Fix Signed-off-by: Itxaka <[email protected]> --------- Signed-off-by: Itxaka <[email protected]>
kairos-io · Dec 20, 2024 · 1a00acf · 1a00acf
1 parent 88a6cb7
commit 1a00acf
Show file tree

Hide file tree

Showing 18 changed files with 1,219 additions and 389 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -61,7 +61,7 @@ jobs:
           tags: auroraboot:latest
       - name: Run e2e tests
         run: |
-          sudo go run github.com/onsi/ginkgo/v2/ginkgo -v -r -p --covermode=atomic --coverprofile=coverage.out --timeout=2h --label-filter "!bootable" ./e2e
+          sudo go run github.com/onsi/ginkgo/v2/ginkgo -v -r -p --covermode=atomic --coverprofile=coverage.out --timeout=2h --label-filter "!bootable && !raw-bootable" ./e2e
       - name: Codecov
         uses: codecov/codecov-action@v5
         env:
@@ -78,3 +78,49 @@ jobs:
           fetch-depth: 0
       - name: Build
         run: earthly +test-bootable
+  test-bootable-raw:
+    runs-on: kvm
+    steps:
+      - uses: earthly/[email protected]
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install deps
+        run: |
+          sudo apt-get update && sudo apt-get install -y ovmf libvirt-clients libvirt-daemon-system libvirt-daemon virtinst bridge-utils qemu-system-x86 qemu-system-x86 qemu-utils qemu-kvm acl udev
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@master
+      - name: Build local image
+        uses: docker/build-push-action@v6
+        with:
+          push: false
+          load: true
+          builder: ${{ steps.buildx.outputs.name }}
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64
+          tags: auroraboot:latest
+      - name: Generate raw image
+        run: |
+          docker run --rm --privileged -v $PWD/build/:/output auroraboot:latest --debug --set "disable_http_server=true" --set "disable_netboot=true" --set "disk.raw=true" --set "disk.size=16000" --set "container_image=quay.io/kairos/ubuntu:24.04-core-amd64-generic-v3.2.3" --set "state_dir=/output"
+      - name: Test raw image is bootable
+        env:
+          FIRMWARE: /usr/share/OVMF/OVMF_CODE.fd
+          USE_QEMU: true
+          MEMORY: 4000
+          CPUS: 2
+          CREATE_VM: true
+          KVM: true
+        run: |
+          # TODO: permissions stuff should be fixed on aurora side, the output dir should contain only
+          # there required output image/iso and not the leftovers from the build process
+          sudo chmod -R 777 build
+          #earthly +test-bootable-raw
+          export RAW_IMAGE=$(find $PWD/build -name *.raw)
+          go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "raw-bootable" -v --fail-fast -r ./e2e
diff --git a/Dockerfile b/Dockerfile
@@ -57,16 +57,12 @@ RUN luet install -y arm-vendor-blob/u-boot-rockchip --system-target /pinebookpro
 ## Odroid fw
 RUN luet install -y firmware/odroid-c2 --system-target /firmware/odroid-c2
 
-## RAW images for current arch
-RUN luet install -y static/grub-efi --system-target /raw/grub
-RUN luet install -y static/grub-config --system-target /raw/grubconfig
-RUN luet install -y static/grub-artifacts --system-target /raw/grubartifacts
-
 ## RAW images for arm64
 # Luet will install this artifacts from the current arch repo, so in x86 it will
 # get them from the x86 repo and we want it to do it from the arm64 repo, even on x86
 # so we use the arm64 luet config and use that to install those on x86
 # This is being used by the prepare_arm_images.sh and build-arch-image.sh scripts
+# TODO: Remove this when raw image is implemented in go as we should get the artifacts from the rootfs
 RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-efi --system-target /arm/raw/grubefi
 RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-config --system-target /arm/raw/grubconfig
 RUN luet install --config /tmp/luet-arm64.yaml -y static/grub-artifacts --system-target /arm/raw/grubartifacts
@@ -86,18 +82,21 @@ RUN rm -Rf /pinebookpro/u-boot/var/tmp
 RUN rm -Rf /pinebookpro/u-boot/var/cache
 RUN rm -Rf /firmware/odroid-c2/var/tmp
 RUN rm -Rf /firmware/odroid-c2/var/cache
-RUN rm -Rf /raw/grub/var/tmp
-RUN rm -Rf /raw/grub/var/cache
-RUN rm -Rf /raw/grubconfig/var/tmp
-RUN rm -Rf /raw/grubconfig/var/cache
-RUN rm -Rf /raw/grubartifacts/var/tmp
-RUN rm -Rf /raw/grubartifacts/var/cache
 RUN rm -Rf /arm/raw/grubefi/var/tmp
 RUN rm -Rf /arm/raw/grubefi/var/cache
 RUN rm -Rf /arm/raw/grubconfig/var/tmp
 RUN rm -Rf /arm/raw/grubconfig/var/cache
 RUN rm -Rf /arm/raw/grubartifacts/var/tmp
 RUN rm -Rf /arm/raw/grubartifacts/var/cache
+# Remove the var dir if empty
+RUN rm -d /grub2/var || true
+RUN rm -d /efi/var || true
+RUN rm -d /rpi/var || true
+RUN rm -d /pinebookpro/u-boot/var || true
+RUN rm -d /firmware/odroid-c2/var || true
+RUN rm -d /arm/raw/grubefi/var || true
+RUN rm -d /arm/raw/grubconfig/var || true
+RUN rm -d /arm/raw/grubartifacts/var || true
 
 # ISO build config
 COPY ./image-assets/add-cloud-init.sh /add-cloud-init.sh
@@ -112,7 +111,6 @@ COPY ./image-assets/prepare_arm_images.sh /prepare_arm_images.sh
 
 # RAW images helpers
 COPY ./image-assets/gce.sh /gce.sh
-COPY ./image-assets/raw-images.sh /raw-images.sh
 COPY ./image-assets/azure.sh /azure.sh
 COPY ./image-assets/netboot.sh /netboot.sh
 

diff --git a/Earthfile b/Earthfile
@@ -53,5 +53,6 @@ test-bootable:
     ARG MEMORY=4000
     ARG CPUS=2
     ARG CREATE_VM=true
+    ARG EFIVARS_EMPTY=true
     RUN date
     RUN go run github.com/onsi/ginkgo/v2/ginkgo run --label-filter "bootable" -v --fail-fast -r ./e2e
diff --git a/deployer/steps.go b/deployer/steps.go
@@ -2,8 +2,10 @@ package deployer
 
 import (
 	"context"
+	"github.com/kairos-io/AuroraBoot/internal"
 	"os"
 	"path/filepath"
+	"strconv"
 
 	"github.com/kairos-io/AuroraBoot/pkg/ops"
 	"github.com/spectrocloud-labs/herd"
@@ -95,7 +97,7 @@ func (d *Deployer) StepGenRawDisk() error {
 	return d.Add(opGenRawDisk,
 		herd.EnableIf(func() bool { return d.rawDiskIsSet() && d.Config.Disk.ARM == nil && !d.Config.Disk.MBR }),
 		d.imageOrSquashFS(),
-		herd.WithCallback(ops.GenEFIRawDisk(d.tmpRootFs(), d.rawDiskPath())))
+		herd.WithCallback(ops.GenEFIRawDisk(d.tmpRootFs(), d.rawDiskPath(), d.rawDiskSize())))
 }
 
 func (d *Deployer) StepGenMBRRawDisk() error {
@@ -117,14 +119,14 @@ func (d *Deployer) StepConvertGCE() error {
 	return d.Add(opConvertGCE,
 		herd.EnableIf(func() bool { return d.Config.Disk.GCE }),
 		herd.WithDeps(opGenRawDisk),
-		herd.WithCallback(ops.ConvertRawDiskToGCE(d.rawDiskPath(), filepath.Join(d.destination(), "disk.raw.gce"))))
+		herd.WithCallback(ops.ConvertRawDiskToGCE(d.rawDiskPath())))
 }
 
 func (d *Deployer) StepConvertVHD() error {
 	return d.Add(opConvertVHD,
 		herd.EnableIf(func() bool { return d.Config.Disk.VHD }),
 		herd.WithDeps(opGenRawDisk),
-		herd.WithCallback(ops.ConvertRawDiskToVHD(d.rawDiskPath(), filepath.Join(d.destination(), "disk.raw.vhd"))))
+		herd.WithCallback(ops.ConvertRawDiskToVHD(d.rawDiskPath())))
 }
 
 func (d *Deployer) StepGenARMImages() error {
@@ -229,8 +231,9 @@ func (d *Deployer) cloudConfigPath() string {
 	return filepath.Join(d.destination(), "config.yaml")
 }
 
+// Return only the path to the output dir, the image name is generated based on the rootfs
 func (d *Deployer) rawDiskPath() string {
-	return filepath.Join(d.destination(), "disk.raw")
+	return d.destination()
 }
 
 func (d *Deployer) diskImgPath() string {
@@ -268,3 +271,17 @@ func (d *Deployer) netbootOption() bool {
 	// squashfs, kernel, and initrd names are tied to the output of /netboot.sh (op.ExtractNetboot)
 	return !d.Config.DisableNetboot
 }
+
+func (d *Deployer) rawDiskSize() uint64 {
+	// parse the string into a uint64
+	// the size is in Mb
+	if d.Config.Disk.Size == "" {
+		return 0
+	}
+	sizeInt, err := strconv.ParseUint(d.Config.Disk.Size, 10, 64)
+	if err != nil {
+		internal.Log.Logger.Error().Err(err).Str("arg", d.Config.Disk.Size).Msg("Failed to parse disk size, setting value to 0")
+		return 0
+	}
+	return sizeInt
+}
diff --git a/e2e/assets/efivars.fd b/e2e/assets/efivars.fd
diff --git a/e2e/bootable_test.go b/e2e/bootable_test.go
@@ -109,8 +109,8 @@ func defaultVMOpts(stateDir string) []types.MachineOption {
 func defaultVMOptsNoDrives(stateDir string) []types.MachineOption {
 	var err error
 
-	if os.Getenv("ISO") == "" && os.Getenv("CREATE_VM") == "true" {
-		fmt.Println("ISO missing")
+	if (os.Getenv("ISO") == "" && os.Getenv("RAW_IMAGE") == "") && os.Getenv("CREATE_VM") == "true" {
+		fmt.Println("ISO or RAW_IMAGE missing")
 		os.Exit(1)
 	}
 
@@ -135,7 +135,6 @@ func defaultVMOptsNoDrives(stateDir string) []types.MachineOption {
 
 	opts := []types.MachineOption{
 		types.QEMUEngine,
-		types.WithISO(os.Getenv("ISO")),
 		types.WithMemory(memory),
 		types.WithCPU(cpus),
 		types.WithSSHPort(strconv.Itoa(sshPort)),
@@ -186,21 +185,37 @@ func defaultVMOptsNoDrives(stateDir string) []types.MachineOption {
 				m.Args = append(m.Args, "-drive",
 					fmt.Sprintf("file=%s,if=pflash,format=raw,readonly=on", FW),
 				)
-
-				// Copy the empty efivars to not modify it
-				f, err := os.ReadFile(filepath.Join(getwd, "assets/efivars.empty.fd"))
-				if err != nil {
-					return err
-				}
-				err = os.WriteFile(filepath.Join(stateDir, "efivars.empty.fd"), f, os.ModePerm)
-				if err != nil {
-					return err
+				// Efivars empty is to boot in setup mode, good for testing UKI and auto enrollment
+				if os.Getenv("EFIVARS_EMPTY") == "true" {
+					// Copy the empty efivars to not modify it
+					f, err := os.ReadFile(filepath.Join(getwd, "assets/efivars.empty.fd"))
+					if err != nil {
+						return err
+					}
+					err = os.WriteFile(filepath.Join(stateDir, "efivars.empty.fd"), f, os.ModePerm)
+					if err != nil {
+						return err
+					}
+
+					m.Args = append(m.Args, "-drive",
+						fmt.Sprintf("file=%s,if=pflash,format=raw", filepath.Join(stateDir, "efivars.empty.fd")),
+					)
+				} else {
+					// This uses the efivars.fd file that has the default keys from Microsoft, useful to test secureboot out of the box
+					f, err := os.ReadFile(filepath.Join(getwd, "assets/efivars.fd"))
+					if err != nil {
+						return err
+					}
+					err = os.WriteFile(filepath.Join(stateDir, "efivars.fd"), f, os.ModePerm)
+					if err != nil {
+						return err
+					}
+
+					m.Args = append(m.Args, "-drive",
+						fmt.Sprintf("file=%s,if=pflash,format=raw", filepath.Join(stateDir, "efivars.fd")),
+					)
 				}
 
-				m.Args = append(m.Args, "-drive",
-					fmt.Sprintf("file=%s,if=pflash,format=raw", filepath.Join(stateDir, "efivars.empty.fd")),
-				)
-
 				// Needed to be set for secureboot!
 				m.Args = append(m.Args, "-machine", "q35,smm=on")
 			}
@@ -209,6 +224,12 @@ func defaultVMOptsNoDrives(stateDir string) []types.MachineOption {
 		},
 		types.WithDataSource(os.Getenv("DATASOURCE")),
 	}
+	if os.Getenv("ISO") != "" {
+		opts = append(opts, types.WithISO(os.Getenv("ISO")))
+	}
+	if os.Getenv("RAW_IMAGE") != "" {
+		opts = append(opts, types.WithDrive(os.Getenv("RAW_IMAGE")))
+	}
 	if os.Getenv("KVM") != "" {
 		opts = append(opts, func(m *types.MachineConfig) error {
 			m.Args = append(m.Args,