Fix Docker Hub images build & push (#4114)

* Fix beta deployment * Fix scripts * No server for flavors * test fix * Fix trivy call * More tests * [MegaLinter] Apply linters fixes * Force prune * Fix call to trivy action * trivy call * Fix release jobs * dbg * linters * trivy --------- Co-authored-by: nvuillam <[email protected]>
oxsecurity · Oct 13, 2024 · 0a4ce21 · 0a4ce21
1 parent d7a1ca5
commit 0a4ce21
Show file tree

Hide file tree

Showing 13 changed files with 201 additions and 155 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -78,6 +78,7 @@ Available commands can be listed with the help command by posting the following
 /help
 ```
 Which returns:
+>
 > Command | Description
 > --- | ---
 > /build | Updates the Dockerfile, documentation, and other files from the yml descriptors

diff --git a/.github/workflows/deploy-ALPHA-flavors.yml b/.github/workflows/deploy-ALPHA-flavors.yml
@@ -142,7 +142,7 @@ jobs:
 
       # Copy ghrc.io image to Docker Hub
       - name: Pull image from GHCR
-        run: docker pull ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:alpha
+        run: docker system prune -a --force && docker pull ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:alpha
       - name: Tag image for Docker Hub
         run: docker tag ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:alpha oxsecurity/megalinter-worker-${{ matrix.flavor }}:alpha
       - name: Push image to Docker Hub

diff --git a/.github/workflows/deploy-ALPHA.yml b/.github/workflows/deploy-ALPHA.yml
@@ -150,7 +150,7 @@ jobs:
 
       # Copy ghrc.io image to Docker Hub
       - name: Pull image from GHCR
-        run: docker pull ghcr.io/oxsecurity/megalinter:alpha
+        run: docker system prune -a --force && docker system prune -a --force && docker pull ghcr.io/oxsecurity/megalinter:alpha
       - name: Tag image for Docker Hub
         run: docker tag ghcr.io/oxsecurity/megalinter:alpha oxsecurity/megalinter:alpha
       - name: Push image to Docker Hub

diff --git a/.github/workflows/deploy-BETA-flavors.yml b/.github/workflows/deploy-BETA-flavors.yml
@@ -19,6 +19,7 @@ on:
     branches:
       - "main"
       - "FlavoredMegaLinters"
+      - dbgbeta
     paths:
       - ".github/workflows/**"
       - "Dockerfile"
@@ -130,7 +131,7 @@ jobs:
         id: meta-w-dhub
         with:
           images: |
-            name=${{ github.repository }}-worker-${{ matrix.flavor }},enable=false
+            name=docker.io/${{ github.repository }}-worker-${{ matrix.flavor }}
           flavor: |
             latest=false
             prefix=beta
@@ -201,35 +202,27 @@ jobs:
       - name: Debug output
         run: "echo \"Tag steps.meta.outputs.tags: ${{ steps.meta.outputs.tags }}\""
       - name: Pull image from GHCR
-        run: docker pull ${{ steps.meta.outputs.tags }}
+        run: docker system prune -a --force && docker pull "${{ steps.meta.outputs.tags }}"
       - name: Tag image for Docker Hub
-        run: docker tag ${{ steps.meta.outputs.tags }} ${{ steps.meta-dhub.outputs.tags }}
+        run: docker tag "${{ steps.meta.outputs.tags }}" "${{ steps.meta-dhub.outputs.tags }}"
       - name: Push image to Docker Hub
-        run: docker push ${{ steps.meta-dhub.outputs.tags }}
-
-      # Copy ghrc.io image to Docker Hub (server)
-      - name: Pull image from GHCR
-        run: docker pull ${{ steps.meta-s.outputs.tags }}
-      - name: Tag image for Docker Hub
-        run: docker tag ${{ steps.meta-s.outputs.tags }} ${{ steps.meta-s-dhub.outputs.tags }}
-      - name: Push image to Docker Hub
-        run: docker push ${{ steps.meta-s-dhub.outputs.tags }}
+        run: docker push "${{ steps.meta-dhub.outputs.tags }}"
 
       # Copy ghrc.io image to Docker Hub (worker)
       - name: Pull image from GHCR
-        run: docker pull ${{ steps.meta-w.outputs.tags }}
+        run: docker system prune -a --force && docker pull "${{ steps.meta-w.outputs.tags }}"
       - name: Tag image for Docker Hub
-        run: docker tag ${{ steps.meta-w.outputs.tags }} ${{ steps.meta-w-dhub.outputs.tags }}
+        run: docker tag "${{ steps.meta-w.outputs.tags }}" "${{ steps.meta-w-dhub.outputs.tags }}"
       - name: Push image to Docker Hub
-        run: docker push ${{ steps.meta-w-dhub.outputs.tags }}
+        run: docker push "${{ steps.meta-w-dhub.outputs.tags }}"
 
       ##############################################
       # Check Docker image security with Trivy #
       ##############################################
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "docker.io/${{ fromJson(steps.meta.outputs.json).tags[0]}}"
+          image-ref: "${{ fromJson(steps.meta.outputs.json).tags[0]}}"
           format: "table"
           exit-code: "1"
           ignore-unfixed: true

diff --git a/.github/workflows/deploy-BETA-linters.yml b/.github/workflows/deploy-BETA-linters.yml
@@ -16,6 +16,7 @@ on:
   push:
     branches:
       - "main"
+      - "dbgbeta"
     paths:
       - ".github/workflows/**"
       - "Dockerfile"
@@ -208,7 +209,7 @@ jobs:
         id: meta-dhub
         with:
           images: |
-            ${{ github.repository }}-only-${{ matrix.linter }}
+            docker.io/${{ github.repository }}-only-${{ matrix.linter }}
           flavor: |
             latest=false
             prefix=beta
@@ -268,20 +269,22 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       # Copy ghrc.io image to Docker Hub (main image)
+      - name: Debug output
+        run: "echo \"Tag steps.meta.outputs.tags: ${{ fromJson(steps.meta-dhub.outputs.json).tags[0]}}\""
       - name: Pull image from GHCR
-        run: docker pull ${{ steps.meta.outputs.tags }}
+        run: docker system prune -a --force && docker pull "${{ fromJson(steps.meta.outputs.json).tags[0]}}"
       - name: Tag image for Docker Hub
-        run: docker tag ${{ steps.meta.outputs.tags }} ${{ steps.meta-dhub.outputs.tags }}
+        run: docker tag "${{ fromJson(steps.meta.outputs.json).tags[0]}}" "${{ fromJson(steps.meta-dhub.outputs.json).tags[0]}}"
       - name: Push image to Docker Hub
-        run: docker push ${{ steps.meta-dhub.outputs.tags }}
+        run: docker push "${{ fromJson(steps.meta-dhub.outputs.json).tags[0]}}"
 
       ##############################################
       # Check Docker image security with Trivy #
       ##############################################
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "docker.io/${{ fromJson(steps.meta.outputs.json).tags[0] }}"
+          image-ref: "${{ fromJson(steps.meta-dhub.outputs.json).tags[0]}}"
           format: "table"
           exit-code: "1"
           ignore-unfixed: true

diff --git a/.github/workflows/deploy-BETA.yml b/.github/workflows/deploy-BETA.yml
@@ -17,7 +17,8 @@ name: "Build & Deploy - BETA"
 on:
   push:
     branches:
-      - "main"
+      - main
+      - dbgbeta
     paths:
       - ".github/workflows/**"
       - "Dockerfile"
@@ -92,7 +93,7 @@ jobs:
         id: meta-dhub
         with:
           images: |
-            name=${{ github.repository }}
+            name=docker.io/${{ github.repository }}
           tags: |
             type=raw,value=beta
 
@@ -110,7 +111,7 @@ jobs:
         id: meta-s-dhub
         with:
           images: |
-            name=${{ github.repository }}-server,enable=false
+            name=docker.io/${{ github.repository }}-server
           tags: |
             type=raw,value=beta
 
@@ -128,7 +129,7 @@ jobs:
         id: meta-w-dhub
         with:
           images: |
-            name=${{ github.repository }}-worker,enable=false
+            name=docker.io/${{ github.repository }}-worker
           tags: |
             type=raw,value=beta
 
@@ -202,27 +203,27 @@ jobs:
 
       # Copy ghrc.io image to Docker Hub (main image)
       - name: Pull image from GHCR
-        run: docker pull ${{ steps.meta.outputs.tags }}
+        run: docker system prune -a --force && docker pull "${{ steps.meta.outputs.tags }}"
       - name: Tag image for Docker Hub
-        run: docker tag ${{ steps.meta.outputs.tags }} ${{ steps.meta-dhub.outputs.tags }}
+        run: docker tag "${{ steps.meta.outputs.tags }}" "${{ steps.meta-dhub.outputs.tags }}"
       - name: Push image to Docker Hub
-        run: docker push ${{ steps.meta-dhub.outputs.tags }}
+        run: docker push "${{ steps.meta-dhub.outputs.tags }}"
 
       # Copy ghrc.io image to Docker Hub (server)
       - name: Pull image from GHCR
-        run: docker pull ${{ steps.meta-s.outputs.tags }}
+        run: docker system prune -a --force && docker pull "${{ steps.meta-s.outputs.tags }}"
       - name: Tag image for Docker Hub
-        run: docker tag ${{ steps.meta-s.outputs.tags }} ${{ steps.meta-s-dhub.outputs.tags }}
+        run: docker tag "${{ steps.meta-s.outputs.tags }}" "${{ steps.meta-s-dhub.outputs.tags }}"
       - name: Push image to Docker Hub
-        run: docker push ${{ steps.meta-s-dhub.outputs.tags }}
+        run: docker push "${{ steps.meta-s-dhub.outputs.tags }}"
 
       # Copy ghrc.io image to Docker Hub (worker)
       - name: Pull image from GHCR
-        run: docker pull ${{ steps.meta-w.outputs.tags }}
+        run: docker system prune -a --force && docker pull "${{ steps.meta-w.outputs.tags }}"
       - name: Tag image for Docker Hub
-        run: docker tag ${{ steps.meta-w.outputs.tags }} ${{ steps.meta-w-dhub.outputs.tags }}
+        run: docker tag "${{ steps.meta-w.outputs.tags }}" "${{ steps.meta-w-dhub.outputs.tags }}"
       - name: Push image to Docker Hub
-        run: docker push ${{ steps.meta-w-dhub.outputs.tags }}
+        run: docker push "${{ steps.meta-w-dhub.outputs.tags }}"
 
       # ###############################
       # # Run tests for code coverage #
@@ -241,7 +242,7 @@ jobs:
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "docker.io/${{ fromJson(steps.meta.outputs.json).tags[0]}}"
+          image-ref: "${{ steps.meta-dhub.outputs.tags }}"
           format: 'table'
           exit-code: '1'
           ignore-unfixed: true

diff --git a/.github/workflows/deploy-DEV-linters.yml b/.github/workflows/deploy-DEV-linters.yml
@@ -224,7 +224,7 @@ jobs:
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "docker.io/${{ fromJson(steps.meta.outputs.json).tags[0] }}"
+          image-ref: "${{ fromJson(steps.meta.outputs.json).tags[0] }}"
           format: "table"
           exit-code: "1"
           ignore-unfixed: true

diff --git a/.github/workflows/deploy-DEV.yml b/.github/workflows/deploy-DEV.yml
@@ -241,7 +241,7 @@ jobs:
       # - name: Run Trivy vulnerability scanner
       #   uses: aquasecurity/trivy-action@master
       #   with:
-      #     image-ref: "docker.io/${{ fromJson(steps.meta.outputs.json).tags[0]}}"
+      #     image-ref: "${{ fromJson(steps.meta.outputs.json).tags[0]}}"
       #     format: 'table'
       #     exit-code: '1'
       #     ignore-unfixed: true

diff --git a/.github/workflows/deploy-RELEASE-flavors.yml b/.github/workflows/deploy-RELEASE-flavors.yml
@@ -76,12 +76,6 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v4
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -108,9 +102,6 @@ jobs:
           secrets: |
             GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
           tags: |
-            docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:v8
-            docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:${{ github.event.release.tag_name }}
-            docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:latest
             ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:v8
             ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:${{ github.event.release.tag_name }}
             ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:latest
@@ -131,20 +122,61 @@ jobs:
           secrets: |
             GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
           tags: |
-            docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:v8
-            docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:${{ github.event.release.tag_name }}
-            docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:latest
             ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:v8
             ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:${{ github.event.release.tag_name }}
             ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:latest
 
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # Copy ghrc.io image to Docker Hub (main image)
+      - name: Pull image from GHCR
+        run: docker system prune -a --force && docker pull ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:v8
+
+      - name: Tag image for Docker Hub
+        run: docker tag ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:v8 docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:v8
+      - name: Push image to Docker Hub v8
+        run: docker push docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:v8
+
+      - name: Tag image for Docker Hub
+        run: docker tag ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:v8 docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:${{ github.event.release.tag_name }}
+      - name: Push image to Docker Hub ${{ github.event.release.tag_name }}
+        run: docker push docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:${{ github.event.release.tag_name }}
+
+      - name: Tag image for Docker Hub
+        run: docker tag ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:v8 docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:latest
+      - name: Push image to Docker Hub latest
+        run: docker push docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:latest
+
+      # Copy ghrc.io image to Docker Hub (worker)
+      - name: Pull image from GHCR
+        run: docker system prune -a --force && docker pull ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:v8
+
+      - name: Tag image for Docker Hub
+        run: docker tag ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:v8 docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:v8
+      - name: Push image to Docker Hub v8
+        run: docker push docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:v8
+
+      - name: Tag image for Docker Hub
+        run: docker tag ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:v8 docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:${{ github.event.release.tag_name }}
+      - name: Push image to Docker Hub ${{ github.event.release.tag_name }}
+        run: docker push docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:${{ github.event.release.tag_name }}
+
+      - name: Tag image for Docker Hub
+        run: docker tag ghcr.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:v8 docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:latest
+      - name: Push image to Docker Hub latest
+        run: docker push docker.io/oxsecurity/megalinter-worker-${{ matrix.flavor }}:latest
+
       ##############################################
       # Check Docker image security with Trivy #
       ##############################################
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: 'docker.io/oxsecurity/megalinter-${{ matrix.flavor }}:${{ github.event.release.tag_name }}'
+          image-ref: 'ghcr.io/oxsecurity/megalinter-${{ matrix.flavor }}:${{ github.event.release.tag_name }}'
           format: 'table'
           exit-code: '1'
           ignore-unfixed: true

diff --git a/.github/workflows/deploy-RELEASE-linters.yml b/.github/workflows/deploy-RELEASE-linters.yml
@@ -182,12 +182,6 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v4
 
-      - name: Login to Docker Hub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -214,14 +208,10 @@ jobs:
           secrets: |
             GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}
           tags: |
-            docker.io/oxsecurity/megalinter-only-${{ matrix.linter }}:v8
-            docker.io/oxsecurity/megalinter-only-${{ matrix.linter }}:${{ github.event.release.tag_name }}
-            docker.io/oxsecurity/megalinter-only-${{ matrix.linter }}:latest
             ghcr.io/oxsecurity/megalinter-only-${{ matrix.linter }}:v8
             ghcr.io/oxsecurity/megalinter-only-${{ matrix.linter }}:${{ github.event.release.tag_name }}
             ghcr.io/oxsecurity/megalinter-only-${{ matrix.linter }}:latest
 
-
       #####################################
       # Run Linter test cases #
       #####################################
@@ -243,7 +233,7 @@ jobs:
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: 'docker.io/oxsecurity/megalinter-only-${{ matrix.linter }}:${{ github.event.release.tag_name }}'
+          image-ref: 'ghcr.io/oxsecurity/megalinter-only-${{ matrix.linter }}:${{ github.event.release.tag_name }}'
           format: 'table'
           exit-code: '1'
           ignore-unfixed: true