From 9f957154f2938137d880a40ecb31c89866eec43b Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Tue, 11 Oct 2022 12:28:05 -0400 Subject: [PATCH 1/6] Add initial draft for building WMCore images via CI/CD --- .github/workflows/images.yaml | 107 ++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 .github/workflows/images.yaml diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml new file mode 100644 index 0000000000..eb1cb0a676 --- /dev/null +++ b/.github/workflows/images.yaml @@ -0,0 +1,107 @@ +name: Build + +on: + push: + tags: + - '*.*.*' + +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Get the Ref + id: get-ref + uses: ankitvgupta/ref-to-tag-action@master + with: + ref: ${{ github.ref }} + head_ref: ${{ github.head_ref }} + + - name: Build wmagent image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/wmagent/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/wmagent + + - name: Build reqmon image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmon/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/reqmon + + - name: Build reqmgr2ms-output image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-output/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-output + + - name: Build reqmgr2ms-rulecleaner image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-rulecleaner/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.rulecleaners.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-rulecleaner + + - name: Build reqmgr2ms-transferor image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-transferor/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.transferors.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-transferor + + - name: Build reqmgr2ms-monitor image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-monitor/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.monitors.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-monitor + + - name: Build reqmgr2ms-unmerged image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-unmerged/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.unmergeds.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-unmerged + +# - name: Login to registry.cern.ch +# uses: docker/login-action@v1.6.0 +# with: +# registry: registry.cern.ch +# username: ${{ secrets.CERN_LOGIN }} +# password: ${{ secrets.CERN_TOKEN }} + +# - name: Publish image to registry.cern.ch +# uses: docker/build-push-action@v1 +# with: +# username: ${{ secrets.CERN_LOGIN }} +# password: ${{ secrets.CERN_TOKEN }} +# registry: registry.cern.ch +# repository: cmsweb/wmarchive +# tag_with_ref: true + +# - name: Login to docker github registry +# uses: docker/login-action@v1.6.0 +# with: +# registry: docker.pkg.github.com +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} + +# - name: Login to Registry +# uses: docker/login-action@v1.6.0 +# with: +# registry: docker.pkg.github.com +# username: ${{ github.actor }} +# password: ${{ secrets.GITHUB_TOKEN }} + +# - name: Push new image to k8s +# run: | +# curl -ksLO https://raw.githubusercontent.com/vkuznet/imagebot/main/imagebot.sh +# sed -i -e "s,COMMIT,${{github.sha}},g" -e "s,REPOSITORY,${{github.repository}},g" -e "s,NAMESPACE,wma,g" -e "s,TAG,${{steps.get-ref.outputs.tag}},g" -e "s,IMAGE,registry.cern.ch/cmsweb/wmarchive,g" -e "s,SERVICE,wmarchive,g" -e "s,HOST,${{secrets.IMAGEBOT_URL}},g" imagebot.sh +# chmod +x imagebot.sh +# cat imagebot.sh +# sh ./imagebot.sh From 0d1835e20073695efb038a949c4d6868a9fc8d40 Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Thu, 13 Oct 2022 10:42:46 -0400 Subject: [PATCH 2/6] Fix steps.get-ref parts --- .github/workflows/images.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index eb1cb0a676..2cb5a4e63a 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -43,28 +43,28 @@ jobs: - name: Build reqmgr2ms-rulecleaner image run: | curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-rulecleaner/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.rulecleaners.tag}},g" Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile cat Dockerfile docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-rulecleaner - name: Build reqmgr2ms-transferor image run: | curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-transferor/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.transferors.tag}},g" Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile cat Dockerfile docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-transferor - name: Build reqmgr2ms-monitor image run: | curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-monitor/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.monitors.tag}},g" Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile cat Dockerfile docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-monitor - name: Build reqmgr2ms-unmerged image run: | curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-unmerged/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.unmergeds.tag}},g" Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile cat Dockerfile docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-unmerged From fa51a70c1a5a61785b65b7677d4b6b053a2806a7 Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Fri, 14 Oct 2022 13:02:57 -0400 Subject: [PATCH 3/6] Added workqueue and t0_reqmon --- .github/workflows/images.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 2cb5a4e63a..97ae0624d7 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -26,6 +26,13 @@ jobs: cat Dockerfile docker build . --tag registry.cern.ch/cmsweb/wmagent + - name: Build workqueue image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/workqueue/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/workqueue + - name: Build reqmon image run: | curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmon/Dockerfile @@ -33,6 +40,13 @@ jobs: cat Dockerfile docker build . --tag registry.cern.ch/cmsweb/reqmon + - name: Build t0_reqmon image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/t0_reqmon/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/t0_reqmon + - name: Build reqmgr2ms-output image run: | curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-output/Dockerfile From 82904e8375050eff23a7b8c8f32fe45e0c10652e Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Fri, 14 Oct 2022 13:07:08 -0400 Subject: [PATCH 4/6] Added reqmgr2 service --- .github/workflows/images.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 97ae0624d7..b9402e0444 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -47,6 +47,13 @@ jobs: cat Dockerfile docker build . --tag registry.cern.ch/cmsweb/t0_reqmon + - name: Build reqmgr2 image + run: | + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2/Dockerfile + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile + cat Dockerfile + docker build . --tag registry.cern.ch/cmsweb/reqmgr2 + - name: Build reqmgr2ms-output image run: | curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-output/Dockerfile From 07cb8e9e60d2aea5438c4b8cf4ed4e319b7dba8d Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Fri, 14 Oct 2022 14:03:15 -0400 Subject: [PATCH 5/6] Adjust tagging convention to match other build workflows --- .github/workflows/images.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index b9402e0444..5bfe531d15 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -3,7 +3,8 @@ name: Build on: push: tags: - - '*.*.*' + - '*' + - '!JENKINS*' jobs: From 18e98d5a940bbebb802f61e0899a5f27a0c58872 Mon Sep 17 00:00:00 2001 From: Valentin Kuznetsov Date: Fri, 28 Oct 2022 13:56:41 -0400 Subject: [PATCH 6/6] Switch from Dockerfile to Dockerfile.dist --- .github/workflows/images.yaml | 60 +++++++++++++++++------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/images.yaml b/.github/workflows/images.yaml index 5bfe531d15..19c9d0a9d3 100644 --- a/.github/workflows/images.yaml +++ b/.github/workflows/images.yaml @@ -22,72 +22,72 @@ jobs: - name: Build wmagent image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/wmagent/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/wmagent/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/wmagent - name: Build workqueue image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/workqueue/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/workqueue/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/workqueue - name: Build reqmon image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmon/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmon/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/reqmon - name: Build t0_reqmon image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/t0_reqmon/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/t0_reqmon/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/t0_reqmon - name: Build reqmgr2 image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/reqmgr2 - name: Build reqmgr2ms-output image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-output/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-output/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-output - name: Build reqmgr2ms-rulecleaner image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-rulecleaner/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-rulecleaner/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-rulecleaner - name: Build reqmgr2ms-transferor image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-transferor/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-transferor/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-transferor - name: Build reqmgr2ms-monitor image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-monitor/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-monitor/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-monitor - name: Build reqmgr2ms-unmerged image run: | - curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-unmerged/Dockerfile - sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile - cat Dockerfile + curl -ksLO https://raw.githubusercontent.com/dmwm/CMSKubernetes/master/docker/pypi/reqmgr2ms-unmerged/Dockerfile.dist + sed -i -e "s,ENV TAG=.*,ENV TAG=${{steps.get-ref.outputs.tag}},g" Dockerfile.dist + cat Dockerfile.dist docker build . --tag registry.cern.ch/cmsweb/reqmgr2ms-unmerged # - name: Login to registry.cern.ch