From 0473cfcbcbc0b07bd00cbae3dc33c1929c53ca37 Mon Sep 17 00:00:00 2001 From: Patrick Schork <354473+pschork@users.noreply.github.com> Date: Mon, 27 May 2024 15:40:33 -0700 Subject: [PATCH 1/8] Update node makefile --- GitVersion.yml | 29 +++++++++++++++++++++++++++++ node/Makefile | 20 ++++++++++++++------ 2 files changed, 43 insertions(+), 6 deletions(-) create mode 100644 GitVersion.yml diff --git a/GitVersion.yml b/GitVersion.yml new file mode 100644 index 000000000..e596af732 --- /dev/null +++ b/GitVersion.yml @@ -0,0 +1,29 @@ +increment: None +branches: + main: + mode: ContinuousDelivery + tag: pre + increment: Patch + prevent-increment-of-merged-branch-version: true + track-merge-target: false + regex: ^master$|^main$ + source-branches: + - release + tracks-release-branches: true + is-release-branch: false + is-mainline: true + pre-release-weight: 55000 + release: + mode: ContinuousDelivery + tag: rc + increment: None + prevent-increment-of-merged-branch-version: true + track-merge-target: false + regex: ^releases?[/-] + source-branches: + - main + - release + tracks-release-branches: false + is-release-branch: true + is-mainline: false + pre-release-weight: 30000 diff --git a/node/Makefile b/node/Makefile index 5b91ed151..6b2715211 100644 --- a/node/Makefile +++ b/node/Makefile @@ -1,12 +1,20 @@ +ifeq ($(wildcard ../.git/*),) +$(warning semver disabled - building from release zip) +GITCOMMIT := "" +GITDATE := "" +SEMVER := $(shell basename $(CURDIR)) +else GITCOMMIT := $(shell git rev-parse --short HEAD) GITDATE := $(shell git log -1 --format=%cd --date=unix) - -# GitVersion provides the semantic versioning for the project. -SEMVER := $(shell docker run --rm --volume "${PWD}/../:/repo" gittools/gitversion:5.12.0 /repo -output json -showvariable SemVer) +SEMVER := $(shell docker run --rm --volume "$(PWD)/../:/repo" gittools/gitversion:5.12.0 /repo -output json -showvariable SemVer) ifeq ($(SEMVER), ) -SEMVER = "0.0.0" # Fallback if docker is not installed or gitversion fails +$(warning semver disabled - docker not installed) +SEMVER := "0.0.0" +endif endif +RELEASE_TAG := $(or $(RELEASE_TAG),latest) + build: clean go mod tidy go build -o ./bin/node ./cmd @@ -17,10 +25,10 @@ clean: docker: docker-node docker-plugin docker-node: - cd ../ && docker build --build-arg SEMVER=${SEMVER} --build-arg GITCOMMIT=${GITCOMMIT} --build-arg GITDATE=${GITDATE} . -t opr-node:${SEMVER} -t opr-node:latest -f node/cmd/Dockerfile + cd ../ && docker build --build-arg SEMVER=${SEMVER} --build-arg GITCOMMIT=${GITCOMMIT} --build-arg GITDATE=${GITDATE} . -t opr-node:${SEMVER} -t opr-node:${RELEASE_TAG} -f node/cmd/Dockerfile docker-plugin: - cd ../ && docker build --build-arg SEMVER=${SEMVER} --build-arg GITCOMMIT=${GITCOMMIT} --build-arg GITDATE=${GITDATE} . -t opr-nodeplugin:${SEMVER} -t opr-nodeplugin:latest -f node/plugin/cmd/Dockerfile + cd ../ && docker build --build-arg SEMVER=${SEMVER} --build-arg GITCOMMIT=${GITCOMMIT} --build-arg GITDATE=${GITDATE} . -t opr-nodeplugin:${SEMVER} -t opr-nodeplugin:${RELEASE_TAG} -f node/plugin/cmd/Dockerfile semver: echo "${SEMVER}" From ee818801760960ab867ebc2585f6189c56f91981 Mon Sep 17 00:00:00 2001 From: Patrick Schork <354473+pschork@users.noreply.github.com> Date: Fri, 31 May 2024 13:06:32 -0700 Subject: [PATCH 2/8] Add release workflow --- .github/workflows/docker-publish-release.yaml | 71 +++++++++++++++++++ Makefile | 26 +++++++ docker-compose-release.yaml | 12 ++++ 3 files changed, 109 insertions(+) create mode 100644 .github/workflows/docker-publish-release.yaml create mode 100644 docker-compose-release.yaml diff --git a/.github/workflows/docker-publish-release.yaml b/.github/workflows/docker-publish-release.yaml new file mode 100644 index 000000000..d2b809e98 --- /dev/null +++ b/.github/workflows/docker-publish-release.yaml @@ -0,0 +1,71 @@ +name: docker-publish-release + +on: + push: + branches: + - release/* + pull_request: + workflow_dispatch: + +env: + REGISTRY: ghcr.io + CACHE-FROM: /tmp/.buildx-cache + CACHE-TO: /tmp/.buildx-cache-new + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install GitVersion + uses: gittools/actions/gitversion/setup@v1.1.1 + with: + versionSpec: '5.x' + + - name: Determine SemVer + uses: gittools/actions/gitversion/execute@v1.1.1 + with: + useConfigFile: true + + - run: | + echo "SemVer ${{ env.semVer }} FullSemVer ${{ env.fullSemVer }}" + name: Display SemVer + + - name: Setup Buildx + uses: docker/setup-buildx-action@v1 + with: + install: true + driver-opts: image=moby/buildkit:master + + - name: Cache docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + if: ${{ success() }} + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + if: ${{ success() }} + + # Build And Push Image + - name: Build docker image release + run: make docker-release-build + if: ${{ success() }} + + - name: Push docker image release + run: make docker-release-push + if: ${{ success() }} diff --git a/Makefile b/Makefile index 97420a3ee..c40ad271c 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,22 @@ .PHONY: compile-el compile-dl clean protoc lint build unit-tests integration-tests-churner integration-tests-indexer integration-tests-inabox integration-tests-inabox-nochurner integration-tests-graph-indexer +ifeq ($(wildcard .git/*),) +$(warning semver disabled - building from release zip) +GITCOMMIT := "" +GITDATE := "" +SEMVER := $(shell basename $(CURDIR)) +else +GITCOMMIT := $(shell git rev-parse --short HEAD) +GITDATE := $(shell git log -1 --format=%cd --date=unix) +SEMVER := $(shell docker run --rm --volume "$(PWD):/repo" gittools/gitversion:5.12.0 /repo -output json -showvariable SemVer) +ifeq ($(SEMVER), ) +$(warning semver disabled - docker not installed) +SEMVER := "0.0.0" +endif +endif + +RELEASE_TAG := $(or $(RELEASE_TAG),latest) + PROTOS := ./api/proto PROTOS_DISPERSER := ./disperser/api/proto PROTO_GEN := ./api/grpc @@ -75,3 +92,12 @@ integration-tests-graph-indexer: integration-tests-dataapi: make dataapi-build go test -v ./disperser/dataapi + +docker-release-build: + RELEASE_TAG=${SEMVER} docker compose -f docker-compose-release.yaml build --build-arg SEMVER=${SEMVER} --build-arg GITCOMMIT=${GITCOMMIT} --build-arg GITDATE=${GITDATE} + +docker-release-push: + RELEASE_TAG=${SEMVER} docker compose -f docker-compose-release.yaml push + +semver: + echo "${SEMVER}" diff --git a/docker-compose-release.yaml b/docker-compose-release.yaml new file mode 100644 index 000000000..8b567e19b --- /dev/null +++ b/docker-compose-release.yaml @@ -0,0 +1,12 @@ +# This file is used for building and pushing images +services: + node: + build: + context: . + dockerfile: node/cmd/Dockerfile + image: ghcr.io/layr-labs/eigenda/node:${RELEASE_TAG} + nodeplugin: + build: + context: . + dockerfile: node/plugin/cmd/Dockerfile + image: ghcr.io/layr-labs/eigenda/nodeplugin:${RELEASE_TAG} From a8e1fe75b3fb6ec08f725cbf89cf5d692d335082 Mon Sep 17 00:00:00 2001 From: Patrick Schork <354473+pschork@users.noreply.github.com> Date: Fri, 31 May 2024 13:10:18 -0700 Subject: [PATCH 3/8] Testing workflow --- .github/workflows/docker-publish-release.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish-release.yaml b/.github/workflows/docker-publish-release.yaml index d2b809e98..4985735b6 100644 --- a/.github/workflows/docker-publish-release.yaml +++ b/.github/workflows/docker-publish-release.yaml @@ -3,6 +3,7 @@ name: docker-publish-release on: push: branches: + - pschork/semver - release/* pull_request: workflow_dispatch: From b7d1263d3705129bdaae2f4fb480a37fbb60c611 Mon Sep 17 00:00:00 2001 From: Patrick Schork <354473+pschork@users.noreply.github.com> Date: Fri, 31 May 2024 13:18:30 -0700 Subject: [PATCH 4/8] Update release docker repository --- .github/workflows/docker-publish-release.yaml | 2 +- docker-compose-release.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish-release.yaml b/.github/workflows/docker-publish-release.yaml index 4985735b6..f63e42eb7 100644 --- a/.github/workflows/docker-publish-release.yaml +++ b/.github/workflows/docker-publish-release.yaml @@ -69,4 +69,4 @@ jobs: - name: Push docker image release run: make docker-release-push - if: ${{ success() }} + if: github.event_name == 'workflow_dispatch' diff --git a/docker-compose-release.yaml b/docker-compose-release.yaml index 8b567e19b..a6e74e4ec 100644 --- a/docker-compose-release.yaml +++ b/docker-compose-release.yaml @@ -4,9 +4,9 @@ services: build: context: . dockerfile: node/cmd/Dockerfile - image: ghcr.io/layr-labs/eigenda/node:${RELEASE_TAG} + image: ghcr.io/layr-labs/eigenda/opr-node:${RELEASE_TAG} nodeplugin: build: context: . dockerfile: node/plugin/cmd/Dockerfile - image: ghcr.io/layr-labs/eigenda/nodeplugin:${RELEASE_TAG} + image: ghcr.io/layr-labs/eigenda/opr-nodeplugin:${RELEASE_TAG} From 1d3b524f724f791d50eebd735a3e18837f3efbf4 Mon Sep 17 00:00:00 2001 From: Patrick Schork <354473+pschork@users.noreply.github.com> Date: Fri, 31 May 2024 22:14:33 -0700 Subject: [PATCH 5/8] Fix tag regex --- .github/workflows/docker-publish-release.yaml | 12 ++++++++++-- GitVersion.yml | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-publish-release.yaml b/.github/workflows/docker-publish-release.yaml index f63e42eb7..d2d503742 100644 --- a/.github/workflows/docker-publish-release.yaml +++ b/.github/workflows/docker-publish-release.yaml @@ -4,9 +4,15 @@ on: push: branches: - pschork/semver - - release/* + tags: + - v* pull_request: workflow_dispatch: + inputs: + ref: + description: "Branch or SHA" + required: false + type: string env: REGISTRY: ghcr.io @@ -23,6 +29,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: + ref: ${{ github.event.inputs.ref }} fetch-depth: 0 - name: Install GitVersion @@ -67,6 +74,7 @@ jobs: run: make docker-release-build if: ${{ success() }} + # Publish if manually triggered or when semver tag is pushed - name: Push docker image release run: make docker-release-push - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') diff --git a/GitVersion.yml b/GitVersion.yml index e596af732..52981c4ac 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -19,7 +19,7 @@ branches: increment: None prevent-increment-of-merged-branch-version: true track-merge-target: false - regex: ^releases?[/-] + regex: ^v*|^releases?[/-] source-branches: - main - release From 97ee5b7f6c35acd796069b98aa47c58ee573c7e1 Mon Sep 17 00:00:00 2001 From: Patrick Schork <354473+pschork@users.noreply.github.com> Date: Sun, 2 Jun 2024 19:29:40 -0700 Subject: [PATCH 6/8] Add force flag to workflow for manually triggered untagged releases --- .github/workflows/docker-publish-release.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-publish-release.yaml b/.github/workflows/docker-publish-release.yaml index d2d503742..a5995b728 100644 --- a/.github/workflows/docker-publish-release.yaml +++ b/.github/workflows/docker-publish-release.yaml @@ -9,10 +9,11 @@ on: pull_request: workflow_dispatch: inputs: - ref: - description: "Branch or SHA" + force: + description: "Force untagged release (expert mode)" required: false - type: string + default: false + type: boolean env: REGISTRY: ghcr.io @@ -29,7 +30,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - ref: ${{ github.event.inputs.ref }} fetch-depth: 0 - name: Install GitVersion @@ -43,7 +43,7 @@ jobs: useConfigFile: true - run: | - echo "SemVer ${{ env.semVer }} FullSemVer ${{ env.fullSemVer }}" + echo "SemVer ${{ env.fullSemVer }} Forced ${{ github.event.inputs.force }}" name: Display SemVer - name: Setup Buildx @@ -74,7 +74,7 @@ jobs: run: make docker-release-build if: ${{ success() }} - # Publish if manually triggered or when semver tag is pushed + # Publish if release is tagged or force == true - name: Push docker image release run: make docker-release-push - if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/v') + if: startsWith(github.ref, 'refs/tags/v') || github.event.inputs.force == 'true' From 492d65b915b065471d6e0884721706172f37595c Mon Sep 17 00:00:00 2001 From: Patrick Schork <354473+pschork@users.noreply.github.com> Date: Mon, 3 Jun 2024 11:52:34 -0700 Subject: [PATCH 7/8] Remove old publish workflow --- .../docker-publish-opr-node-images.yaml | 99 ------------------- 1 file changed, 99 deletions(-) delete mode 100644 .github/workflows/docker-publish-opr-node-images.yaml diff --git a/.github/workflows/docker-publish-opr-node-images.yaml b/.github/workflows/docker-publish-opr-node-images.yaml deleted file mode 100644 index ed3aed9cc..000000000 --- a/.github/workflows/docker-publish-opr-node-images.yaml +++ /dev/null @@ -1,99 +0,0 @@ -name: Push Public Docker images to GHCR with Caching - -on: - workflow_dispatch: - inputs: - commit_sha: - description: 'Specific Commit SHA (Required)' - required: true - version: - description: 'Version (Required)' - required: true - gitcommit: - description: 'GitCommit (Required)' - required: true - release_tag: - description: 'Release Tag (Optional)' - required: false - default: '' - -env: - REGISTRY: ghcr.io - CACHE-FROM: /tmp/.buildx-cache - CACHE-TO: /tmp/.buildx-cache-new - -jobs: - build-and-push: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Checkout repository at specified commit - uses: actions/checkout@v3 - with: - ref: ${{ github.event.inputs.commit_sha }} - - name: Get Commit Date - id: get_date - run: | - GIT_DATE=$(git log -1 --format=%cd --date=format:'%Y-%m-%d' ${{ github.event.inputs.gitcommit }} || date '+%Y-%m-%d') - echo "GIT_DATE=$GIT_DATE" >> $GITHUB_ENV - echo "::set-output name=gitDate::$GIT_DATE" - - - name: Setup Buildx - uses: docker/setup-buildx-action@v1 - with: - install: true - driver-opts: image=moby/buildkit:master - - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.event.inputs.commit_sha }} - restore-keys: | - ${{ runner.os }}-buildx- - if: ${{ success() }} - - - name: Log into registry ${{ env.REGISTRY }} - uses: docker/login-action@v2 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - if: ${{ success() }} - - - name: Set Tag Name - id: set_tag - run: echo "tag=${{ github.event.inputs.release_tag || github.event.inputs.commit_sha }}" >> $GITHUB_OUTPUT - if: ${{ success() }} - - - name: Build and Push Opr Node Image - uses: docker/build-push-action@v2 - with: - file: ./node/cmd/Dockerfile - push: true - tags: ${{ env.REGISTRY }}/layr-labs/eigenda/opr-node:${{ steps.set_tag.outputs.tag }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - build-args: SEMVER=${{ github.event.inputs.version }},GITCOMMIT=${{ github.event.inputs.gitcommit }},GITDATE=${{ steps.get_date.outputs.gitDate }} - if: ${{ success() }} - - - name: Build and Push NodePlugin Image - uses: docker/build-push-action@v2 - with: - context: . - file: ./node/plugin/cmd/Dockerfile - push: true - tags: ${{ env.REGISTRY }}/layr-labs/eigenda/opr-nodeplugin:${{ steps.set_tag.outputs.tag }} - cache-from: type=local,src=${{ env.CACHE-FROM }} - cache-to: type=local,dest=${{ env.CACHE-TO }} - if: ${{ success() }} - - - name: Update cache - uses: actions/cache@v2 - with: - path: ${{ env.CACHE-TO }} - key: ${{ runner.os }}-buildx-${{ steps.set_tag.outputs.tag }} - restore-keys: | - ${{ runner.os }}-buildx- \ No newline at end of file From 07cd3ccd8c21db600e5b652170c29222fead60d5 Mon Sep 17 00:00:00 2001 From: Patrick Schork <354473+pschork@users.noreply.github.com> Date: Mon, 3 Jun 2024 16:21:14 -0700 Subject: [PATCH 8/8] Remove dev branch from workflow --- .github/workflows/docker-publish-release.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/docker-publish-release.yaml b/.github/workflows/docker-publish-release.yaml index a5995b728..60f19ed02 100644 --- a/.github/workflows/docker-publish-release.yaml +++ b/.github/workflows/docker-publish-release.yaml @@ -2,8 +2,6 @@ name: docker-publish-release on: push: - branches: - - pschork/semver tags: - v* pull_request: