feat(docker): compile toolchains separately #1127
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| # Test built docker images by building simple projects inside them | |
| name: dagger | |
| on: | |
| pull_request: | |
| paths: | |
| - '.dagger-ci' | |
| - '.github/workflows/docker-build-and-test.yml' | |
| - 'docker/**' | |
| - 'tests/**' | |
| push: | |
| branches: ['main'] | |
| paths: | |
| - '.dagger-ci' | |
| - '.github/workflows/docker-build-and-test.yml' | |
| - 'docker/**' | |
| - 'tests/**' | |
| release: | |
| schedule: | |
| # First day of the month at midnight | |
| - cron: '0 0 1 * 0' | |
| workflow_dispatch: | |
| env: | |
| REGISTRY: ghcr.io | |
| permissions: | |
| contents: read | |
| jobs: | |
| #============================= | |
| # Dynamically generate matrix | |
| #============================= | |
| get-matrix: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: get-matrix | |
| id: get-matrix | |
| run: | | |
| # Disable SC2046: Quote this to prevent word splitting | |
| # I can't quote this, just look at it | |
| # shellcheck disable=SC2046 | |
| echo matrix=$( yq --input-format yaml --output-format json '.services | keys[]' docker/compose.yaml | sed 's/"//g' | jq -Rs 'split("\n") | del(.[-1])' | jq -c ) >> "${GITHUB_OUTPUT}" | |
| - name: Check | |
| run: | | |
| jq . <<< '${{ steps.get-matrix.outputs.matrix }}' | |
| outputs: | |
| matrix: ${{ steps.get-matrix.outputs.matrix }} | |
| get-matrix-coreboot: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: get-matrix | |
| id: get-matrix | |
| run: | | |
| # shellcheck disable=SC2046 | |
| echo matrix=$( yq --input-format yaml --output-format json '.services | keys[] | select(. | test("coreboot.*"))' docker/compose.yaml | sed 's/"//g' | jq -Rs 'split("\n") | del(.[-1])' | jq -c ) >> "${GITHUB_OUTPUT}" | |
| - name: Check | |
| run: | | |
| jq . <<< '${{ steps.get-matrix.outputs.matrix }}' | |
| outputs: | |
| matrix: ${{ steps.get-matrix.outputs.matrix }} | |
| #===================== | |
| # Coreboot toolchains | |
| #===================== | |
| build-coreboot-toolchains: | |
| timeout-minutes: 120 | |
| needs: | |
| - get-matrix-coreboot | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: ['amd64', 'arm64'] | |
| dockerfile: ${{ fromJson(needs.get-matrix-coreboot.outputs.matrix) }} | |
| runs-on: ${{ matrix.arch == 'arm64' && 'ARM64' || 'ubuntu-latest' }} | |
| container: | |
| # At the time of writing (2024-10) we cannot use ubuntu:noble as it is broken | |
| image: ubuntu:jammy | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| # Use coreboot mirrors | |
| BUILDGCC_OPTIONS: -m | |
| steps: | |
| - name: Install dependencies for CI | |
| run: | | |
| apt-get update | |
| apt-get install -y --no-install-recommends \ | |
| ca-certificates \ | |
| curl \ | |
| git \ | |
| jq \ | |
| sudo \ | |
| tzdata \ | |
| upx-ucl \ | |
| wget | |
| update-ca-certificates | |
| - name: Get yq | |
| # the --no-check-certificate is needed because GitHub | |
| run: | | |
| wget -q --no-check-certificate -O /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_${{ matrix.arch }} | |
| chmod 755 /usr/local/bin/yq | |
| - name: Configure tzdata | |
| run: | | |
| dpkg-reconfigure --frontend noninteractive tzdata | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Get coreboot version | |
| id: version | |
| run: | | |
| yq -r '.services.["${{ matrix.dockerfile }}"].build.args[] | select(test("COREBOOT_VERSION=.*"))' docker/compose.yaml >> "${GITHUB_OUTPUT}" | |
| - name: Clone coreboot | |
| run: | | |
| git clone --depth 1 "https://review.coreboot.org/coreboot.git" -b "${{ steps.version.outputs.COREBOOT_VERSION }}" | |
| - name: Get coreboot commit hash | |
| id: coreboot-hash | |
| run: | | |
| cd coreboot | |
| COREBOOT_HASH="$( git rev-parse --short HEAD )" | |
| echo "${COREBOOT_HASH}" | |
| echo "COREBOOT_HASH=${COREBOOT_HASH}" >> "${GITHUB_OUTPUT}" | |
| - name: Artefact and cache key | |
| id: cache-key | |
| run: | | |
| CACHE_KEY="coreboot-${{ steps.version.outputs.COREBOOT_VERSION }}-${{ steps.coreboot-hash.outputs.COREBOOT_HASH }}-${{ matrix.arch }}" | |
| echo "${CACHE_KEY}" | |
| echo "CACHE_KEY=${CACHE_KEY}" >> "${GITHUB_OUTPUT}" | |
| - name: Restore cached toolchains | |
| id: cache-toolchains | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: coreboot/util/crossgcc/xgcc | |
| key: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc | |
| - name: Restore cached utils | |
| id: cache-utils | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: /usr/local/bin | |
| key: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils | |
| - name: Debug list crossgcc | |
| if: steps.cache-toolchains.outputs.cache-hit == 'true' | |
| run: | | |
| ls -a1lh coreboot/util/crossgcc/xgcc | |
| - name: Debug list utils | |
| if: steps.cache-utils.outputs.cache-hit == 'true' | |
| run: | | |
| ls -a1lh /usr/local/bin | |
| - name: Install dependencies if needed | |
| if: steps.cache-toolchains.outputs.cache-hit != 'true' | |
| run: | | |
| apt-get install -y --no-install-recommends \ | |
| acpica-tools \ | |
| bc \ | |
| bison \ | |
| bsdmainutils \ | |
| build-essential \ | |
| flex \ | |
| gnat \ | |
| imagemagick \ | |
| libelf-dev \ | |
| libncurses5-dev \ | |
| libnss3-dev \ | |
| libssl-dev \ | |
| m4 \ | |
| nasm \ | |
| openssh-client \ | |
| pkgconf \ | |
| python-is-python3 \ | |
| python3-pip \ | |
| qemu-system-x86 \ | |
| upx-ucl \ | |
| uuid-dev \ | |
| zlib1g-dev | |
| - name: Install dependencies if needed (amd64) | |
| if: matrix.arch == 'amd64' && steps.cache-toolchains.outputs.cache-hit != 'true' | |
| run: | | |
| apt-get install -y --no-install-recommends \ | |
| iucode-tool | |
| - name: Build coreboot toolchains | |
| if: steps.cache-toolchains.outputs.cache-hit != 'true' | |
| run: | | |
| cd coreboot | |
| make crossgcc CPUS="$(nproc)" | |
| - name: Build coreboot utils | |
| if: steps.cache-utils.outputs.cache-hit != 'true' | |
| run: | | |
| cd coreboot | |
| make -C util/ifdtool install | |
| make -C util/cbfstool install | |
| - name: Compress toolchain binaries | |
| # This step should shrink the size of single toolchain from 1.5 GB down to around 700 MB | |
| # I think it is save to compress all binaries except libraries, hence the '-wholename' | |
| if: steps.cache-toolchains.outputs.cache-hit != 'true' | |
| run: | | |
| cd coreboot/util/crossgcc/xgcc | |
| # shellcheck disable=SC2016 | |
| find . -type f -wholename '*/bin/*' -exec bash -c 'upx-ucl -9 "$1"' shell {} \; || true | |
| # Store toolchains and utils in cache | |
| - name: Cache toolchains | |
| uses: actions/cache/save@v4 | |
| if: steps.cache-toolchains.outputs.cache-hit != 'true' | |
| with: | |
| path: coreboot/util/crossgcc/xgcc | |
| key: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc | |
| - name: Cache utils | |
| uses: actions/cache/save@v4 | |
| if: steps.cache-utils.outputs.cache-hit != 'true' | |
| with: | |
| path: /usr/local/bin | |
| key: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils | |
| # Upload toolchains and utils as artifacts | |
| - name: Tar toolchain to prevent permission loss | |
| # Docs: https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss | |
| run: | | |
| if [ ! -f "coreboot/util/crossgcc/xgcc-tar/${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar" ]; then | |
| mv coreboot/util/crossgcc/xgcc coreboot/util/crossgcc/${{ matrix.arch }}-xgcc | |
| tar -cf ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar coreboot/util/crossgcc/${{ matrix.arch }}-xgcc | |
| fi | |
| - name: Upload toolchain | |
| uses: actions/[email protected] | |
| with: | |
| name: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc | |
| path: ${{ steps.cache-key.outputs.CACHE_KEY }}-xgcc.tar | |
| retention-days: 30 | |
| include-hidden-files: true | |
| compression-level: 9 | |
| - name: Upload utils | |
| uses: actions/[email protected] | |
| with: | |
| name: ${{ steps.cache-key.outputs.CACHE_KEY }}-utils | |
| path: /usr/local/bin | |
| retention-days: 30 | |
| include-hidden-files: true | |
| compression-level: 9 | |
| #========================= | |
| # Build Docker containers | |
| #========================= | |
| build: | |
| name: build_test_publish | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| needs: | |
| - get-matrix | |
| - build-coreboot-toolchains | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| dockerfile: ${{ fromJson(needs.get-matrix.outputs.matrix) }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| # We have to use my own fork of actions/delete-package-versions at the moment | |
| # to have access to 'dry-run' and 'ignore-versions-include-tags' features | |
| # We can switch to upstream whe following PRs get merged: | |
| # - [dry-run](https://github.com/actions/delete-package-versions/pull/119/commits) | |
| # - [tags](https://github.com/actions/delete-package-versions/pull/104 | |
| - name: Delete old packages | |
| uses: AtomicFS/delete-package-versions@main | |
| continue-on-error: | |
| true | |
| # we have continue-on-error because when I make a fork of this repo to debug something, | |
| # the Docker containers would not build because this step fails to fetch existing containers | |
| # (in fresh fork there are none) | |
| with: | |
| package-name: firmware-action/${{ matrix.dockerfile }} | |
| package-type: container | |
| min-versions-to-keep: 5 | |
| ignore-versions: | |
| '^(main|latest|v(\d+\.?)+)$' | |
| # ignore: | |
| # - main | |
| # - latest | |
| # - vX | |
| # - vX.X | |
| # - vX.X.X | |
| dry-run: false | |
| ignore-versions-include-tags: true | |
| - name: Setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup docker-compose | |
| uses: KengoTODA/actions-setup-docker-compose@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Validate compose file | |
| run: docker-compose -f docker/compose.yaml config | |
| - name: Install python dependencies | |
| run: pip install -r ./.dagger-ci/daggerci/requirements.txt | |
| - name: Run dagger pipeline | |
| # If building coreboot use 120 minutes timeout, otherwise 15 minutes | |
| timeout-minutes: ${{ startsWith(matrix.dockerfile, 'coreboot_') && 120 || 15 }} | |
| run: | | |
| if [[ "${GITHUB_EVENT_NAME}" == 'release' ]] || [[ "${GITHUB_REF}" == *'main' ]] || [[ "${GITHUB_REF_TYPE}" == 'tag' ]]; then | |
| echo "Enable publishing" | |
| python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }} --publish | |
| else | |
| echo "Disable publishing" | |
| python .dagger-ci/daggerci/main.py -d ${{ matrix.dockerfile }} | |
| fi | |
| shell: bash | |
| env: | |
| GITHUB_REGISTRY: ${{ env.REGISTRY }} | |
| GITHUB_ACTOR: ${{ github.actor }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |