diff --git a/.github/workflows/call-build-images.yaml b/.github/workflows/call-build-images.yaml index 4f8c9b9323f..fdd8ee3396e 100644 --- a/.github/workflows/call-build-images.yaml +++ b/.github/workflows/call-build-images.yaml @@ -15,7 +15,8 @@ on: registry: description: The registry to push container images to. type: string - required: true + required: false + default: ghcr.io username: description: The username for the registry. type: string @@ -33,11 +34,6 @@ on: type: string required: false default: "" - platforms: - description: The platforms to build for - type: string - required: false - default: 'linux/amd64, linux/arm64, linux/arm/v7, linux/s390x' secrets: token: description: The Github token or similar to authenticate with for the registry. @@ -74,25 +70,34 @@ jobs: replace-with: "$1" flags: "g" - # This is the intended approach to multi-arch image and all the other checks scanning, - # signing, etc only trigger from this. - call-build-images: - needs: - - call-build-images-meta - name: Multiarch container images to GHCR - runs-on: ubuntu-latest-8-cores - environment: ${{ inputs.environment }} + # Taken from https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners + # We split this out to make it easier to restart just one of them if it fails and do all in parallel + call-build-single-arch-container-images: + # Allow us to continue to create a manifest if we want + continue-on-error: true permissions: contents: read packages: write - outputs: - production-digest: ${{ steps.build_push.outputs.digest }} - debug-digest: ${{ steps.debug_build_push.outputs.digest }} + strategy: + fail-fast: false + matrix: + platform: + - amd64 + - arm64 + - arm/v7 + - s390x + target: + - production + - debug + name: ${{ matrix.platform }}/${{ matrix.target }} container image build + # Use GitHub Actions ARM hosted runners + runs-on: ${{ (contains(matrix.platform, 'arm') && 'ubuntu-22.04-arm') || 'ubuntu-latest' }} steps: - - name: Checkout code for modern style builds + - name: Checkout code uses: actions/checkout@v4 with: ref: ${{ inputs.ref }} + token: ${{ secrets.token }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -104,29 +109,19 @@ jobs: uses: docker/login-action@v3 with: registry: ${{ inputs.registry }} - username: ${{ inputs.username }} + username: ${{ github.actor }} password: ${{ secrets.token }} - - name: Extract metadata from Github - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ inputs.registry }}/${{ inputs.image }} - tags: | - raw,${{ inputs.version }} - raw,${{ needs.call-build-images-meta.outputs.major-version }} - raw,latest - - - name: Build the production images - id: build_push + - name: Build and push by digest the standard ${{ matrix.target }} image + id: build uses: docker/build-push-action@v6 with: + # Use path context rather than Git context as we want local files file: ./dockerfiles/Dockerfile context: . - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: ${{ inputs.platforms }} - target: production + target: ${{ matrix.target }} + outputs: type=image,name=${{ inputs.registry }}/${{ inputs.image }},push-by-digest=true,name-canonical=true,push=true + platforms: linux/${{ matrix.platform }} # Must be disabled to provide legacy format images from the registry provenance: false push: true @@ -134,7 +129,88 @@ jobs: build-args: | FLB_NIGHTLY_BUILD=${{ inputs.unstable }} RELEASE_VERSION=${{ inputs.version }} + WAMR_BUILD_TARGET=${{ (contains(matrix.platform, 'arm/v7') && 'ARMV7') || '' }} + + - name: Export ${{ matrix.target }} digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + shell: bash + + - name: Upload ${{ matrix.target }} digest + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.target }}-digests-${{ (contains(matrix.platform, 'arm/v7') && 'arm-v7') || matrix.platform }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + # Take the digests and produce a multi-arch manifest from them. + call-build-container-image-manifests: + permissions: + contents: read + packages: write + name: Upload multi-arch container image manifests + runs-on: ubuntu-latest + needs: + - call-build-images-meta + - call-build-single-arch-container-images + outputs: + version: ${{ steps.meta.outputs.version }} + steps: + - name: Extract metadata from Github + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.registry }}/${{ inputs.image }} + tags: | + raw,${{ inputs.version }} + raw,${{ needs.call-build-images-meta.outputs.major-version }} + raw,latest + + - name: Download production digests + uses: actions/download-artifact@v4 + with: + pattern: production-digests-* + path: /tmp/production-digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.token }} + + - name: Create production manifest + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ inputs.registry }}/${{ inputs.image }}@sha256:%s ' *) + shell: bash + working-directory: /tmp/production-digests + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ inputs.registry }}/${{ inputs.image }}:${{ steps.meta.outputs.version }} + shell: bash + + # Take the digests and produce a multi-arch manifest from them. + call-build-debug-container-image-manifests: + permissions: + contents: read + packages: write + name: Upload debug multi-arch container image manifests + runs-on: ubuntu-latest + needs: + - call-build-images-meta + - call-build-single-arch-container-images + outputs: + version: ${{ steps.debug-meta.outputs.version }} + steps: - id: debug-meta uses: docker/metadata-action@v5 with: @@ -144,28 +220,39 @@ jobs: raw,${{ needs.call-build-images-meta.outputs.major-version }}-debug raw,latest-debug - - name: Build the debug multi-arch images - id: debug_build_push - uses: docker/build-push-action@v6 + - name: Download debug digests + uses: actions/download-artifact@v4 with: - file: ./dockerfiles/Dockerfile - context: . - tags: ${{ steps.debug-meta.outputs.tags }} - labels: ${{ steps.debug-meta.outputs.labels }} - platforms: ${{ inputs.platforms }} - # Must be disabled to provide legacy format images from the registry - provenance: false - target: debug - push: true - load: false - build-args: | - FLB_NIGHTLY_BUILD=${{ inputs.unstable }} - RELEASE_VERSION=${{ inputs.version }} + pattern: production-digests-* + path: /tmp/debug-digests + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ github.actor }} + password: ${{ secrets.token }} + + - name: Create debug manifest + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ inputs.registry }}/${{ inputs.image }}@sha256:%s ' *) + shell: bash + working-directory: /tmp/debug-digests + + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ inputs.registry }}/${{ inputs.image }}:${{ steps.debug-meta.outputs.version }} + shell: bash call-build-images-generate-schema: needs: - call-build-images-meta - - call-build-images + - call-build-container-image-manifests runs-on: ubuntu-latest environment: ${{ inputs.environment }} permissions: @@ -195,7 +282,7 @@ jobs: call-build-images-scan: needs: - call-build-images-meta - - call-build-images + - call-build-container-image-manifests name: Trivy + Dockle image scan runs-on: ubuntu-latest environment: ${{ inputs.environment }} @@ -230,7 +317,8 @@ jobs: call-build-images-sign: needs: - call-build-images-meta - - call-build-images + - call-build-container-image-manifests + - call-build-debug-container-image-manifests name: Deploy and sign multi-arch container image manifests permissions: contents: read @@ -251,13 +339,13 @@ jobs: # # We use recursive signing on the manifest to cover all the images. run: | - cosign sign --recursive \ + cosign sign --recursive --force \ -a "repo=${{ github.repository }}" \ -a "workflow=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ -a "ref=${{ github.sha }}" \ -a "release=${{ inputs.version }}" \ - "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.production-digest }}" \ - "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.debug-digest }}" + "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-container-image-manifests.outputs.version }}" \ + "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-debug-container-image-manifests.outputs.version }}" shell: bash # Ensure we move on to key-based signing as well continue-on-error: true @@ -270,13 +358,13 @@ jobs: # The key needs to cope with newlines run: | echo -e "${COSIGN_PRIVATE_KEY}" > /tmp/my_cosign.key - cosign sign --key /tmp/my_cosign.key --recursive \ + cosign sign --key /tmp/my_cosign.key --recursive --force \ -a "repo=${{ github.repository }}" \ -a "workflow=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ -a "ref=${{ github.sha }}" \ -a "release=${{ inputs.version }}" \ - "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.production-digest }}" \ - "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-images.outputs.debug-digest }}" + "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-container-image-manifests.outputs.version }}" \ + "${{ inputs.registry }}/${{ inputs.image }}@${{ needs.call-build-debug-container-image-manifests.outputs.version }}" rm -f /tmp/my_cosign.key shell: bash continue-on-error: true diff --git a/.github/workflows/pr-package-tests.yaml b/.github/workflows/pr-package-tests.yaml index 2f249a5f770..f323ad82371 100644 --- a/.github/workflows/pr-package-tests.yaml +++ b/.github/workflows/pr-package-tests.yaml @@ -34,6 +34,24 @@ jobs: - name: Debug event output uses: hmarr/debug-action@v3 + pr-container-builds: + name: PR - container builds + needs: + - pr-package-test-build-get-meta + - pr-package-test-build-generate-matrix + uses: ./.github/workflows/call-build-images.yaml + with: + version: pr-${{ github.event.number }} + ref: ${{ github.ref }} + registry: ghcr.io + username: ${{ github.actor }} + image: ${{ github.repository }}/pr + unstable: ${{ needs.pr-package-test-build-get-meta.outputs.date }} + secrets: + token: ${{ secrets.GITHUB_TOKEN }} + cosign_private_key: ${{ secrets.COSIGN_PRIVATE_KEY }} + cosign_private_key_password: ${{ secrets.COSIGN_PASSWORD }} + pr-package-test-build-generate-matrix: name: PR - packages build matrix needs: diff --git a/.github/workflows/staging-build.yaml b/.github/workflows/staging-build.yaml index 4ac2274f0e0..a409dd337ac 100644 --- a/.github/workflows/staging-build.yaml +++ b/.github/workflows/staging-build.yaml @@ -79,7 +79,6 @@ jobs: username: ${{ github.actor }} image: ${{ github.repository }}/staging environment: staging - platforms: 'linux/amd64, linux/arm64' secrets: token: ${{ secrets.GITHUB_TOKEN }} cosign_private_key: ${{ secrets.COSIGN_PRIVATE_KEY }} diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile index 5d3d49f23bf..ae0782ce4a6 100644 --- a/dockerfiles/Dockerfile +++ b/dockerfiles/Dockerfile @@ -13,7 +13,7 @@ # docker buildx build --platform "linux/amd64,linux/arm64,linux/arm/v7,linux/s390x" -f ./dockerfiles/Dockerfile.multiarch --build-arg FLB_TARBALL=https://github.com/fluent/fluent-bit/archive/v1.8.11.tar.gz ./dockerfiles/ # Set this to the current release version: it gets done so as part of the release. -ARG RELEASE_VERSION=3.2.5 +ARG RELEASE_VERSION=3.2.4 # For multi-arch builds - assumption is running on an AMD64 host FROM multiarch/qemu-user-static:x86_64-arm AS qemu-arm32 @@ -66,7 +66,16 @@ COPY . ./ # We split the builder setup out so people can target it or use as a base image without doing a full build. FROM builder-base AS builder WORKDIR /src/fluent-bit/build/ -RUN cmake -DFLB_RELEASE=On \ + +# Required to be set to ARMV7 for that target +ARG WAMR_BUILD_TARGET +ARG EXTRA_CMAKE_FLAGS +ENV EXTRA_CMAKE_FLAGS=${EXTRA_CMAKE_FLAGS} + +# We do not want word splitting for EXTRA_CMAKE_FLAGS in case multiple are defined +# hadolint ignore=SC2086 +RUN [ -n "${WAMR_BUILD_TARGET:-}" ] && EXTRA_CMAKE_FLAGS="$EXTRA_CMAKE_FLAGS -DWAMR_BUILD_TARGET=$WAMR_BUILD_TARGET"; \ + cmake -DFLB_RELEASE=On \ -DFLB_JEMALLOC=On \ -DFLB_TLS=On \ -DFLB_SHARED_LIB=Off \ @@ -79,8 +88,12 @@ RUN cmake -DFLB_RELEASE=On \ -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ -DFLB_LOG_NO_CONTROL_CHARS=On \ -DFLB_CHUNK_TRACE="$FLB_CHUNK_TRACE" \ + $EXTRA_CMAKE_FLAGS \ .. +ARG CFLAGS="-v" +ENV CFLAGS=${CFLAGS} + RUN make -j "$(getconf _NPROCESSORS_ONLN)" RUN install bin/fluent-bit /fluent-bit/bin/ diff --git a/packaging/distros/raspbian/Dockerfile b/packaging/distros/raspbian/Dockerfile index 2d266da8a73..64c55200a91 100644 --- a/packaging/distros/raspbian/Dockerfile +++ b/packaging/distros/raspbian/Dockerfile @@ -34,6 +34,19 @@ RUN apt-get update && \ libsasl2-2 libsasl2-dev libyaml-dev libcurl4-openssl-dev pkg-config && \ apt-get install -y --reinstall lsb-base lsb-release +# raspbian/bookworm base image +FROM balenalib/rpi-raspbian:bookworm as raspbian-bookworm-base +ENV DEBIAN_FRONTEND noninteractive + +# hadolint ignore=DL3008,DL3015 +RUN apt-get update && \ + apt-get install -y curl ca-certificates build-essential \ + cmake make bash sudo wget unzip dh-make \ + libsystemd-dev zlib1g-dev flex bison \ + libssl3 libssl-dev libpq-dev postgresql-server-dev-all \ + libsasl2-2 libsasl2-dev libyaml-dev libcurl4-openssl-dev pkg-config && \ + apt-get install -y --reinstall lsb-base lsb-release + # Common build for all distributions now # hadolint ignore=DL3006 FROM $BASE_BUILDER as builder @@ -58,6 +71,8 @@ ARG FLB_OUT_KAFKA=On ARG FLB_OUT_PGSQL=On ARG FLB_JEMALLOC=On ARG FLB_CHUNK_TRACE=On +#Tell raspbian packages should be using armv7. +ARG WAMR_BUILD_TARGET=ARMV7A ENV CFLAGS=$CFLAGS RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ @@ -71,6 +86,7 @@ RUN cmake -DCMAKE_INSTALL_PREFIX="$CMAKE_INSTALL_PREFIX" \ -DFLB_NIGHTLY_BUILD="$FLB_NIGHTLY_BUILD" \ -DFLB_JEMALLOC="${FLB_JEMALLOC}" \ -DFLB_CHUNK_TRACE="${FLB_CHUNK_TRACE}" \ + -DWAMR_BUILD_TARGET="${WAMR_BUILD_TARGET}" \ ../ VOLUME [ "/output" ]