Release #83
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| push: | |
| tags: | |
| - "v*" | |
| schedule: | |
| # Release at 00:00 UTC+8 | |
| - cron: "0 16 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| tags: | |
| description: The tags to be released | |
| required: false | |
| type: string | |
| permissions: | |
| id-token: write | |
| pull-requests: write | |
| checks: write | |
| statuses: write | |
| contents: write | |
| env: | |
| BUILD_PROFILE: release | |
| jobs: | |
| create_release: | |
| name: create release | |
| runs-on: ubuntu-latest | |
| outputs: | |
| version: ${{ steps.generated-tag.outputs.tag }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get latest tag | |
| id: get-latest-tag | |
| run: | | |
| echo "tag=`gh release list -L 1 | cut -f 1`" >> $GITHUB_OUTPUT | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Bump version | |
| id: generated-tag | |
| uses: actions/github-script@v6 | |
| with: | |
| script: | | |
| if (context.ref.startsWith("refs/tags/")) { | |
| let tag = context.ref.replace("refs/tags/", ""); | |
| core.setOutput('tag', tag); | |
| console.log(`This event pushed a tag ${tag}, return directly.`) | |
| return | |
| } | |
| if ("${{ github.event.inputs.tags }}") { | |
| let tag = "${{ github.event.inputs.tags }}"; | |
| core.setOutput('tag', tag); | |
| console.log(`This event triggered by workflow_dispatch with a tag ${tag}, return directly.`) | |
| return | |
| } | |
| let tag = "${{ steps.get-latest-tag.outputs.tag }}"; | |
| let result = /v(\d+)\.(\d+)\.(\d+)/g.exec(tag); | |
| if (result === null) { | |
| throw `The previous tag ${{ steps.get-latest-tag.outputs.tag }} is invalid, ignoring`; | |
| } | |
| let major = result[1]; | |
| let minor = result[2]; | |
| let patch = (parseInt(result[3]) + 1).toString(); | |
| let next_tag = `v${major}.${minor}.${patch}-nightly`; | |
| console.log(`This event is triggered, return generated ${next_tag}.`) | |
| core.setOutput('tag', next_tag) | |
| - name: Create github release if not exist | |
| # Only create release when the tag is not exist | |
| if: 'steps.generated-tag.outputs.tag != steps.get-latest-tag.outputs.tag' | |
| # Allow this action failure | |
| # continue-on-error: true | |
| # Reference: https://cli.github.com/manual/gh_release_create | |
| run: | | |
| echo "Create a release for ${{ steps.generated-tag.outputs.tag }}" | |
| gh release create ${{ steps.generated-tag.outputs.tag }} --generate-notes -p | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| macos: | |
| runs-on: macos-11 | |
| needs: create_release | |
| env: | |
| RUNNER_PROVIDER: github | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - x86_64 | |
| - aarch64 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get target | |
| id: target | |
| run: echo 'target=${{ matrix.arch }}-apple-darwin' >> $GITHUB_OUTPUT | |
| - name: Install coreutils for macOS sha256sum | |
| run: brew install coreutils | |
| - name: Build Release | |
| uses: ./.github/actions/build_macos | |
| with: | |
| target: ${{ steps.target.outputs.target }} | |
| artifacts: sqllogictests,metactl,meta,query | |
| upload: false | |
| - name: Basic Sqllogic Test | |
| if: matrix.arch == 'x86_64' | |
| shell: bash | |
| env: | |
| TEST_HANDLERS: http | |
| run: | | |
| mkdir -p target/release | |
| cp ./target/${{ steps.target.outputs.target }}/release/databend-{meta,metactl,query,sqllogictests} ./target/release/ | |
| bash ./scripts/ci/ci-run-sqllogic-tests.sh base | |
| - name: Pack binaries | |
| run: | | |
| brew install gnu-tar | |
| sudo /usr/sbin/purge | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| mkdir -p release/${target}/{bin,configs,scripts} | |
| cp ./target/${target}/release/databend-* release/${target}/bin/ | |
| rm -f release/${target}/bin/*.d | |
| cp ./scripts/distribution/configs/databend-* release/${target}/configs/ | |
| cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt | |
| cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/ | |
| gtar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs scripts readme.txt | |
| - name: generate sha256sums | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt | |
| - name: post sha256 | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sha256sums | |
| path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt | |
| retention-days: 1 | |
| - name: Publish Binaries | |
| uses: ./.github/actions/publish_binary | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ steps.target.outputs.target }} | |
| repo_role_arn: ${{ secrets.REPO_ROLE_ARN }} | |
| linux: | |
| runs-on: [self-hosted, X64, Linux, 16c32g] | |
| needs: create_release | |
| env: | |
| RUNNER_PROVIDER: aws | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - x86_64 | |
| - aarch64 | |
| libc: | |
| - gnu | |
| - musl | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Get target | |
| id: target | |
| run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}' >> $GITHUB_OUTPUT | |
| - name: Build Release | |
| uses: ./.github/actions/build_linux | |
| with: | |
| sha: ${{ github.sha }} | |
| target: ${{ steps.target.outputs.target }} | |
| artifacts: sqllogictests,metactl,meta,query | |
| upload: false | |
| - name: Basic Sqllogic Test | |
| if: matrix.arch == 'x86_64' | |
| shell: bash | |
| env: | |
| TEST_HANDLERS: http | |
| run: | | |
| mkdir -p target/release | |
| cp ./target/${{ steps.target.outputs.target }}/release/databend-{meta,query,sqllogictests} ./target/release/ | |
| bash ./scripts/ci/ci-run-sqllogic-tests.sh base | |
| - name: Pack binaries | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| mkdir -p release/${target}/{bin,configs,systemd,scripts} | |
| cp ./target/${target}/release/databend-* release/${target}/bin/ | |
| rm -f release/${target}/bin/*.d | |
| cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/ | |
| cp ./scripts/distribution/configs/databend-* release/${target}/configs/ | |
| cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt | |
| cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/ | |
| cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/ | |
| tar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs systemd scripts readme.txt | |
| - name: generate sha256sums | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt | |
| - name: post sha256 | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sha256sums | |
| path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt | |
| retention-days: 1 | |
| - name: Publish Binaries | |
| uses: ./.github/actions/publish_binary | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ steps.target.outputs.target }} | |
| repo_role_arn: ${{ secrets.REPO_ROLE_ARN }} | |
| hive: | |
| runs-on: [self-hosted, X64, Linux, 16c32g] | |
| needs: create_release | |
| env: | |
| RUNNER_PROVIDER: aws | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| arch: | |
| - x86_64 | |
| libc: | |
| - gnu | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Get target | |
| id: target | |
| run: echo 'target=${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}' >> $GITHUB_OUTPUT | |
| - name: Setup Build Tool | |
| uses: ./.github/actions/setup_build_tool | |
| with: | |
| target: ${{ steps.target.outputs.target }} | |
| - name: Build Release | |
| uses: ./.github/actions/build_linux | |
| with: | |
| sha: ${{ github.sha }} | |
| target: ${{ steps.target.outputs.target }} | |
| artifacts: query | |
| features: hive | |
| upload: false | |
| - name: Pack binaries | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| mkdir -p release/${target}/{bin,configs,systemd,scripts} | |
| cp ./target/${target}/release/databend-* release/${target}/bin/ | |
| rm -f release/${target}/bin/*.d | |
| cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/ | |
| cp ./scripts/distribution/configs/databend-* release/${target}/configs/ | |
| cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt | |
| cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/ | |
| cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/ | |
| tar -C ./release/${target} -czvf databend-hive-${version}-${target}.tar.gz bin configs systemd scripts readme.txt | |
| - name: generate sha256sums | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version=${{ needs.create_release.outputs.version }} | |
| sha256sum databend-hive-${version}-${target}.tar.gz >> sha256-hive-${version}-${target}.txt | |
| - name: post sha256 | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: sha256sums | |
| path: sha256-hive-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt | |
| retention-days: 1 | |
| - name: Publish Binaries | |
| uses: ./.github/actions/publish_binary | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| version: ${{ needs.create_release.outputs.version }} | |
| target: ${{ steps.target.outputs.target }} | |
| repo_role_arn: ${{ secrets.REPO_ROLE_ARN }} | |
| mode: hive | |
| testsuites: | |
| runs-on: ubuntu-latest | |
| needs: create_release | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Upload to github release | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| version=${{ needs.create_release.outputs.version }} | |
| tar -C ./tests/sqllogictests -czvf testsuites-${version}.tar.gz suites | |
| gh release upload ${version} testsuites-${version}.tar.gz --clobber | |
| docker_combined: | |
| name: docker combined | |
| runs-on: [self-hosted, X64, Linux, 4c8g] | |
| needs: [create_release, linux] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Download binaries for usage | |
| id: download_binaries | |
| run: | | |
| version="${{ needs.create_release.outputs.version }}" | |
| declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu") | |
| mkdir -p ./distro/ | |
| for platform in ${!platform_targets[@]}; do | |
| target=${platform_targets[$platform]} | |
| aws s3 cp s3://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz distro/ --no-progress | |
| mkdir -p ./target/${target}/release | |
| tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/ | |
| mkdir -p ./distro/linux/${platform} | |
| cp ./target/${target}/release/databend-* ./distro/linux/${platform} | |
| done | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - uses: ./.github/actions/setup_docker | |
| id: login | |
| with: | |
| repo: databend | |
| ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
| dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
| dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and push | |
| id: docker_build | |
| uses: docker/build-push-action@v3 | |
| with: | |
| push: true | |
| tags: | | |
| ${{ steps.login.outputs.dockerhub_repo }}:latest | |
| ${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }} | |
| ${{ steps.login.outputs.ecr_repo }}:latest | |
| ${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }} | |
| platforms: linux/amd64,linux/arm64 | |
| context: . | |
| file: ./docker/Dockerfile | |
| - name: Update repo description | |
| continue-on-error: true | |
| uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| repository: ${{ steps.login.outputs.dockerhub_repo }} | |
| short-description: "A modern cloud data warehouse. Also available in the cloud: https://app.databend.com." | |
| readme-filepath: ./docker/README.md | |
| docker_separate: | |
| name: docker separate | |
| runs-on: [self-hosted, X64, Linux, 4c8g] | |
| needs: [create_release, linux] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| service: | |
| - meta | |
| - query | |
| distro: | |
| - debian | |
| - distroless | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Get the version | |
| id: get_version | |
| run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT | |
| - name: Download binaries for usage | |
| id: download_binaries | |
| run: | | |
| version="${{ needs.create_release.outputs.version }}" | |
| declare -A platform_targets=( ["arm64"]="aarch64-unknown-linux-gnu" ["amd64"]="x86_64-unknown-linux-gnu") | |
| mkdir -p ./distro/ | |
| for platform in ${!platform_targets[@]}; do | |
| target=${platform_targets[$platform]} | |
| aws s3 cp s3://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz distro/ --no-progress | |
| mkdir -p ./target/${target}/release | |
| tar x -C ./target/${target}/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/ | |
| mkdir -p ./distro/linux/${platform} | |
| cp ./target/${target}/release/databend-* ./distro/linux/${platform} | |
| done | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| - uses: ./.github/actions/setup_docker | |
| id: login | |
| with: | |
| repo: databend-${{ matrix.service }} | |
| ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} | |
| dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} | |
| dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: get image tags | |
| id: get_image_tags | |
| shell: bash | |
| run: | | |
| _tags="${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}" | |
| _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}-${{ matrix.distro }}" | |
| if [[ "${{ matrix.distro }}" == "debian" ]]; then | |
| _tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:${{ needs.create_release.outputs.version }}" | |
| _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:${{ needs.create_release.outputs.version }}" | |
| _tags="${_tags},${{ steps.login.outputs.dockerhub_repo }}:latest" | |
| _tags="${_tags},${{ steps.login.outputs.ecr_repo }}:latest" | |
| fi | |
| echo "IMAGE_TAGS=${_tags}" >> $GITHUB_OUTPUT | |
| - name: push service image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| push: true | |
| tags: ${{ steps.get_image_tags.outputs.IMAGE_TAGS }} | |
| platforms: linux/amd64,linux/arm64 | |
| context: . | |
| file: ./docker/${{ matrix.distro }}/${{ matrix.service }}.Dockerfile | |
| distribution: | |
| runs-on: [self-hosted, X64, Linux, 4c8g] | |
| needs: [create_release, linux] | |
| strategy: | |
| matrix: | |
| arch: | |
| - x86_64 | |
| - aarch64 | |
| packager: | |
| - deb | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Install nfpm@latest | |
| run: | | |
| curl -sSLo nfpm.tar.gz https://github.com/goreleaser/nfpm/releases/download/v2.26.0/nfpm_2.26.0_Linux_x86_64.tar.gz | |
| tar xf nfpm.tar.gz | |
| sudo mv nfpm /usr/local/bin | |
| sudo chmod a+x /usr/local/bin/nfpm | |
| rm nfpm.tar.gz | |
| - name: Get target | |
| id: target | |
| run: | | |
| echo 'target=${{ matrix.arch }}-unknown-linux-gnu' >> $GITHUB_OUTPUT | |
| - name: Download binaries for usage | |
| id: download_binaries | |
| run: | | |
| target=${{ steps.target.outputs.target }} | |
| version="${{ needs.create_release.outputs.version }}" | |
| mkdir -p ./distro/ | |
| aws s3 cp s3://repo.databend.rs/databend/${version}/databend-${version}-${target}.tar.gz distro/ --no-progress | |
| tar x -C distro -f ./distro/databend-${version}-${target}.tar.gz | |
| - name: Build Packages | |
| id: build_packages | |
| run: | | |
| export name="databend" | |
| export version="${{ needs.create_release.outputs.version }}" | |
| export path="distro" | |
| case "${{ matrix.arch }}" in | |
| x86_64) | |
| export arch="amd64" | |
| ;; | |
| aarch64) | |
| export arch="arm64" | |
| ;; | |
| esac | |
| nfpm pkg --packager ${{ matrix.packager }} -f <(envsubst '${name} ${version} ${path} ${arch}' < scripts/distribution/nfpm.yaml) | |
| - name: Update release to github | |
| shell: bash | |
| # Reference: https://cli.github.com/manual/gh_release_upload | |
| run: | | |
| version="${{ needs.create_release.outputs.version }}" | |
| # name looks like: `databend_0.8.144~nightly_amd64.deb` | |
| gh release upload ${version} databend_*.${{ matrix.packager }} --clobber | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| deb: | |
| runs-on: ubuntu-latest | |
| needs: [create_release, distribution] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: ./.github/actions/publish_deb | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| version: ${{ needs.create_release.outputs.version }} | |
| repo_role_arn: ${{ secrets.REPO_ROLE_ARN }} | |
| gpg_signing_key: ${{ secrets.GPG_KEY_DEB }} | |
| sha256sums: | |
| needs: [create_release, linux, macos, distribution] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v3 | |
| - name: download sha256sums | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: sha256sums | |
| - shell: bash | |
| run: | | |
| for file in *.txt | |
| do | |
| cat ${file} >> sha256sums.txt | |
| done | |
| - name: Upload checksums | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| version="${{ needs.create_release.outputs.version }}" | |
| gh release upload ${version} sha256sums.txt --clobber | |
| benchmark: | |
| needs: [create_release, linux] | |
| uses: ./.github/workflows/reuse.benchmark.yml | |
| secrets: inherit | |
| with: | |
| sha: ${{ github.sha }} | |
| run_id: ${{ github.run_id }} | |
| source: release | |
| source_id: ${{ needs.create_release.outputs.version }} | |
| version: ${{ needs.create_release.outputs.version }} | |
| bindings_python: | |
| needs: create_release | |
| uses: ./.github/workflows/bindings_python.yml | |
| secrets: inherit | |
| with: | |
| tag: ${{ needs.create_release.outputs.version }} | |
| notify: | |
| runs-on: ubuntu-latest | |
| if: always() | |
| needs: | |
| - create_release | |
| - linux | |
| - docker_combined | |
| - docker_separate | |
| - distribution | |
| - deb | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: | | |
| status="${{ (contains(needs.*.result, 'failure') && 'failure') || (contains(needs.*.result, 'cancelled') && 'cancelled') || 'success' }}" | |
| jq -n -f .github/release-report.jq \ | |
| --arg title "[Release] ${{ needs.create_release.outputs.version }}" \ | |
| --arg content "Build result: ${status}" \ | |
| --arg link "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ | |
| > /tmp/release-report.json | |
| curl -X POST "${{ secrets.RELEASE_REPORT_WEBHOOK }}" \ | |
| -H 'Content-Type: application/json' \ | |
| -H 'cache-control: no-cache' \ | |
| -d @/tmp/release-report.json |