Merge pull request opendatahub-io#182 from gmfrasca/ghaction-image-bu…

…ilder

GitHub Image Build Actions

.github/actions/build/action.yaml

@@ -0,0 +1,63 @@
+name: "Create a podman build"
+description: "This workflow can be used to create a podman build and push to quay.io from source branches."
+inputs:
+  IMAGE_REPO:
+    description: "Quay image repo name."
+    required: true
+  DOCKERFILE:
+    description: "Path to Dockerfile."
+    required: true
+  GH_REPO:
+    description: "GH org/repo that contains the dockerfile to source."
+    required: true
+  OVERWRITE:
+    default: "false"
+    description: "GH org/repo that contains the dockerfile to source."
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/checkout@v3
+      with:
+        repository: ${{ inputs.GH_REPO }}
+        ref: ${{ env.SOURCE_BRANCH }}
+        path: build
+    - name: Login to Quay.io
+      uses: redhat-actions/podman-login@v1
+      with:
+        username: ${{ env.QUAY_ID }}
+        password: ${{ env.QUAY_TOKEN }}
+        registry: quay.io
+    # Tags in quay stick around as objects in api, when deleted quay adds "end_ts" time stamp on the tag.
+    # To determine a tag is deleted, we need to check for the presence of this tag.
+    # Also note there can be multiple tags created/deleted, thus we have 4 cases:
+    # Case 1: Only 1 tag was ever created "tags: [{name:..},]"   -- no end_ts field
+    # Case 2: No tag was ever created "tags: []"
+    # Case 3: >1 tags were created, but they were all deleted at some point [{name:..., end_ts,..},....]  -- note they all have "end_ts" field.
+    # Case 4: >1 tags were created, but the most recent one was never deleted (same as case 3, but the latest tag does not have "end_ts".
+    - name: Check if Image already exists
+      shell: bash
+      if: inputs.OVERWRITE == 'false'
+      env:
+        IMAGE: quay.io/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}:${{ env.TARGET_IMAGE_TAG }}
+      run: |
+        echo ${{ inputs.OVERWRITE }}
+
+        tags=$(curl --request GET 'https://quay.io/api/v1/repository/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}/tag/?specificTag=${{ env.TARGET_IMAGE_TAG }}')
+        latest_tag_has_end_ts=$(echo $tags | yq .tags - | yq 'sort_by(.start_ts) | reverse' - -P | yq .[0].end_ts -)
+        notempty=$(echo ${tags} | yq .tags - | yq any)
+
+        # Image only exists if there is a tag that does not have "end_ts" (i.e. it is still present).
+        if [[ "$notempty" == "true" && $latest_tag_has_end_ts == "null" ]]; then
+            echo "::error::The image ${{ env.IMAGE }} already exists"
+            exit 1
+        else
+            echo "Image does not exist...proceeding with build & push."
+        fi
+    - name: Build image
+      shell: bash
+      working-directory: build
+      env:
+        IMAGE: quay.io/${{ env.QUAY_ORG }}/${{ inputs.IMAGE_REPO }}:${{ env.TARGET_IMAGE_TAG }}
+      run: |
+          podman build . -f ${{ inputs.DOCKERFILE }} -t ${{ env.IMAGE }} && podman push ${{ env.IMAGE }}

.github/workflows/build-images.yaml

@@ -0,0 +1,104 @@
+name: Build images from sources.
+run-name: Build images from sources.
+on:
+  workflow_call:
+    inputs:
+      src_branch:
+        type: string
+        default: 'v1.0.x'
+        description: 'Source branch to build DSP from'
+        required: true
+      target_tag:
+        type: string
+        default: 'vx.y.z'
+        description: 'Target Image Tag'
+        required: true
+      quay_org:
+        type: string
+        default: 'opendatahub'
+        description: 'Quay Organization'
+        required: true
+      overwrite_imgs:
+        type: string
+        default: 'true'
+        description: 'Overwrite images in quay if they already exist for this release.'
+        required: true
+      fail_fast:
+        type: string
+        default: 'true'
+        description: 'Stop running entire Workflow if a single build fails'
+        required: true
+
+  workflow_dispatch:
+    inputs:
+      src_branch:
+        type: string
+        default: 'v1.0.x'
+        description: 'Source branch to build DSP from'
+        required: true
+      target_tag:
+        type: string
+        default: 'vx.y.z'
+        description: 'Target Image Tag'
+        required: true
+      quay_org:
+        type: string
+        default: 'opendatahub'
+        description: 'Quay Organization'
+        required: true
+      overwrite_imgs:
+        type: string
+        default: 'true'
+        description: 'Overwrite images in quay if they already exist for this release.'
+        required: true
+      fail_fast:
+        type: string
+        default: 'true'
+        description: 'Stop running entire Workflow if a single build fails'
+        required: true
+env:
+  SOURCE_BRANCH: ${{ inputs.src_branch }}
+  QUAY_ORG: ${{ inputs.quay_org }}
+  QUAY_ID: ${{ secrets.QUAY_ROBOT_USERNAME }}
+  QUAY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  TARGET_IMAGE_TAG: ${{ inputs.target_tag }}
+  OVERWRITE_IMAGES: ${{ inputs.overwrite_imgs }}
+jobs:
+  build-images-with-tag:
+    continue-on-error: false
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: ${{ inputs.fail_fast == 'true' }}
+      matrix:
+        include:
+          - image: ds-pipelines-api-server
+            dockerfile: backend/Dockerfile
+          - image: ds-pipelines-frontend
+            dockerfile: frontend/Dockerfile
+          - image: ds-pipelines-cacheserver
+            dockerfile: backend/Dockerfile.cacheserver
+          - image: ds-pipelines-persistenceagent
+            dockerfile: backend/Dockerfile.persistenceagent
+          - image: ds-pipelines-scheduledworkflow
+            dockerfile: backend/Dockerfile.scheduledworkflow
+          - image: ds-pipelines-viewercontroller
+            dockerfile: backend/Dockerfile.viewercontroller
+          - image: ds-pipelines-artifact-manager
+            dockerfile: backend/artifact_manager/Dockerfile
+          - image: ds-pipelines-metadata-writer
+            dockerfile: backend/metadata_writer/Dockerfile
+          - image: ds-pipelines-metadata-grpc
+            dockerfile: third-party/ml-metadata/Dockerfile
+          - image: ds-pipelines-metadata-envoy
+            dockerfile: third-party/metadata_envoy/Dockerfile
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/build
+        name: Build Image
+        with:
+          IMAGE_REPO: ${{ matrix.image }}
+          DOCKERFILE: ${{ matrix.dockerfile }}
+          GH_REPO: ${{ github.repository }}
+          OVERWRITE: ${{ env.OVERWRITE_IMAGES }}

.github/workflows/build-master.yml

@@ -0,0 +1,73 @@
+name: Build images for Master branch
+on:
+  push:
+    branches:
+      - master
+concurrency:
+  group: ${{ github.workflow }}
+  cancel-in-progress: true
+env:
+  QUAY_ORG: opendatahub
+  QUAY_ID: ${{ secrets.QUAY_ROBOT_USERNAME }}
+  QUAY_TOKEN: ${{ secrets.QUAY_ROBOT_TOKEN }}
+  SOURCE_BRANCH: master
+jobs:
+  build-master-images:
+    continue-on-error: false
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: ds-pipelines-api-server
+            dockerfile: backend/Dockerfile
+          - image: ds-pipelines-frontend
+            dockerfile: frontend/Dockerfile
+          - image: ds-pipelines-cacheserver
+            dockerfile: backend/Dockerfile.cacheserver
+          - image: ds-pipelines-persistenceagent
+            dockerfile: backend/Dockerfile.persistenceagent
+          - image: ds-pipelines-scheduledworkflow
+            dockerfile: backend/Dockerfile.scheduledworkflow
+          - image: ds-pipelines-viewercontroller
+            dockerfile: backend/Dockerfile.viewercontroller
+          - image: ds-pipelines-artifact-manager
+            dockerfile: backend/artifact_manager/Dockerfile
+          - image: ds-pipelines-metadata-writer
+            dockerfile: backend/metadata_writer/Dockerfile
+          - image: ds-pipelines-metadata-grpc
+            dockerfile: third-party/ml-metadata/Dockerfile
+          - image: ds-pipelines-metadata-envoy
+            dockerfile: third-party/metadata_envoy/Dockerfile
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate Tag
+        shell: bash
+        id: tags
+        env:
+          SOURCE_BRANCH: ${{ env.SOURCE_BRANCH }}
+        run: |
+          commit_sha=${{ github.event.after }}
+          tag=${SOURCE_BRANCH}-${commit_sha:0:7}
+          echo "tag=${tag}" >> $GITHUB_OUTPUT
+      - name: Build Image
+        uses: ./.github/actions/build
+        env:
+          IMG: quay.io/${{ env.QUAY_ORG }}/${{ matrix.image }}:${{ steps.tags.outputs.tag }}
+          TARGET_IMAGE_TAG: ${{ steps.tags.outputs.tag }}
+        with:
+          OVERWRITE: true
+          IMAGE_REPO: ${{ matrix.image }}
+          DOCKERFILE: ${{ matrix.dockerfile }}
+          GH_REPO: ${{ github.repository }}
+      - name: Tag latest
+        shell: bash
+        env:
+          IMG: quay.io/${{ env.QUAY_ORG }}/${{ matrix.image }}
+          NEWEST_TAG: ${{ steps.tags.outputs.tag }}
+          SOURCE_BRANCH: ${{ env.SOURCE_BRANCH }}
+        run: |
+          podman tag ${IMG}:${NEWEST_TAG} ${IMG}:latest
+          podman push ${IMG}:latest
+          podman tag ${IMG}:${NEWEST_TAG} ${IMG}:${SOURCE_BRANCH}
+          podman push ${IMG}:${SOURCE_BRANCH}

.github/workflows/build-prs-trigger.yaml

@@ -0,0 +1,33 @@
+name: Trigger build images for PRs
+on:
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'guides/**'
+      - 'images/**'
+      - '**/README.md'
+    types:
+      - opened
+      - reopened
+      - closed
+      - synchronize
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+jobs:
+  upload-data:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Save PR payload
+        shell: bash
+        run: |
+          mkdir -p ./pr
+          echo ${{ github.event.pull_request.number }} >> ./pr/pr_number
+          echo ${{ github.event.pull_request.state }} >> ./pr/pr_state
+          echo ${{ github.event.pull_request.head.sha }} >> ./pr/head_sha
+          echo ${{ github.event.action }} >> ./pr/event_action
+      - uses: actions/upload-artifact@v2
+        with:
+          name: pr
+          path: pr/