From c5e8aaab7a30b7933fe4aac511778cd57b3b6925 Mon Sep 17 00:00:00 2001 From: Leo Cai <143980630+leo-cai-timeplus@users.noreply.github.com> Date: Tue, 14 Nov 2023 09:31:01 +0800 Subject: [PATCH] Feature/issue 231 binary release workflow (#270) update the workflow for release --- .github/workflows/release_build.yml | 150 +++++++++++++++++++++++++++- .github/workflows/run_command.yml | 13 +++ 2 files changed, 158 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml index 2cf54ef20f1..8dde0bf6c68 100644 --- a/.github/workflows/release_build.yml +++ b/.github/workflows/release_build.yml @@ -5,13 +5,14 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: jobs: - Build: + Build_Linux_X86_64: uses: timeplus-io/proton/.github/workflows/run_command.yml@develop with: ec2-instance-type: c5.4xlarge ec2-image-id: ami-042a37e33a285c22b submodules: 'recursive' run_mode: 'start' # start ec2 on demand instance + upload_files: proton-*-Linux-x86_64 command: | cd $GITHUB_WORKSPACE @@ -29,13 +30,21 @@ jobs: rm cache.tar.gz # compiling - ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output + ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then echo "Compiling proton Failed" exit 127 fi + # get proton tag + PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` + echo "Proton tag: $PROTON_TAG" + echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT + + PROTON_BINARY=proton-$PROTON_TAG-Linux-x86_64 + echo "Proton Binary Name: $PROTON_BINARY" + docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx cd $GITHUB_WORKSPACE/docker/server @@ -48,6 +57,8 @@ jobs: # upload build cache tar -zcf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache . aws s3 cp --no-progress ./cache.tar.gz s3://tp-internal/proton/ + + mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -56,13 +67,14 @@ jobs: GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - BuildArm: + Build_Linux_Arm64: uses: timeplus-io/proton/.github/workflows/run_command.yml@develop with: ec2-instance-type: c6g.4xlarge ec2-image-id: ami-0f3dbc4cc9994fdee submodules: 'recursive' run_mode: 'start' # start ec2 on demand instance + upload_files: proton-*-Linux-aarch64 command: | cd $GITHUB_WORKSPACE @@ -80,7 +92,16 @@ jobs: rm cache-arm.tar.gz # compiling - ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output + ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output + + # get proton tag + PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` + echo "Proton tag: $PROTON_TAG" + echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT + + PROTON_BINARY=proton-$PROTON_TAG-Linux-aarch64 + echo "Proton Binary Name: $PROTON_BINARY" + docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx cd $GITHUB_WORKSPACE/docker/server @@ -93,6 +114,8 @@ jobs: # upload build cache tar -zcf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache . aws s3 cp --no-progress ./cache-arm.tar.gz s3://tp-internal/proton/ + + mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY secrets: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -102,7 +125,7 @@ jobs: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} CommitTag: - needs: [Build, BuildArm] + needs: [Build_Linux_X86_64, Build_Linux_Arm64] uses: timeplus-io/proton/.github/workflows/run_command.yml@develop with: ec2-instance-type: c5.4xlarge @@ -154,3 +177,120 @@ jobs: GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + Build_Darwin_X86_64: + uses: timeplus-io/proton/.github/workflows/run_command.yml@develop + with: + ec2-instance-type: c5.4xlarge + ec2-image-id: ami-042a37e33a285c22b + submodules: 'recursive' + run_mode: 'start' # start ec2 on demand instance + upload_files: proton-*-Darwin-x86_64 + command: | + cd $GITHUB_WORKSPACE + + # git config + git config user.name "proton-robot" + git config user.email "proton_robot@timeplus.io" + + # update version + ./release --version patch + + # download the pre-built binary of v8 (this is only for cross-compile) + aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-x64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/ + tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/ + rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz + chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator + chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot + chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque + + # prepare build cache + mkdir $GITHUB_WORKSPACE/ccache + echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf + + # compiling + ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin + + if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then + echo "Compiling proton Failed" + exit 127 + fi + + # clean ccache and build footprint + rm -rf $GITHUB_WORKSPACE/ccache + rm -rf $GITHUB_WORKSPACE/build_docker + + # get proton tag + PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` + echo "Proton tag: $PROTON_TAG" + echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT + + PROTON_BINARY=proton-$PROTON_TAG-Darwin-x86_64 + echo "Proton Binary Name: $PROTON_BINARY" + + mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + Build_Darwin_Arm64: + uses: timeplus-io/proton/.github/workflows/run_command.yml@develop + with: + ec2-instance-type: c6g.4xlarge + ec2-image-id: ami-0f3dbc4cc9994fdee + submodules: 'recursive' + run_mode: 'start' # start ec2 on demand instance + upload_files: proton-*-Darwin-arm64 + command: | + cd $GITHUB_WORKSPACE + + # git config + git config user.name "proton-robot" + git config user.email "proton_robot@timeplus.io" + + # update version + ./release --version patch + + # download the pre-built binary of v8 (this is only for cross-compile) + aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-arm64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/ + tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/ + rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz + chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator + chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot + chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque + + # prepare build cache + mkdir $GITHUB_WORKSPACE/ccache + echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf + + # compiling + ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin-aarch64 + if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then + echo "Compiling proton Failed" + exit 127 + fi + + # clean ccache and build footprint + rm -rf $GITHUB_WORKSPACE/ccache + rm -rf $GITHUB_WORKSPACE/build_docker + + # get proton tag + PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` + echo "Proton tag: $PROTON_TAG" + echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT + + PROTON_BINARY=proton-$PROTON_TAG-Darwin-arm64 + echo "Proton Binary Name: $PROTON_BINARY" + + mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY + secrets: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }} + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/run_command.yml b/.github/workflows/run_command.yml index 3b6e99f27aa..f543eb00d0e 100644 --- a/.github/workflows/run_command.yml +++ b/.github/workflows/run_command.yml @@ -53,6 +53,10 @@ on: required: false type: string default: 'start_spot' # start_spot, start + upload_files: + required: false + type: string + default: '' secrets: AWS_ACCESS_KEY_ID: required: true @@ -152,9 +156,18 @@ jobs: - name: Prepare directory run: mkdir /test_output /artifacts - name: execute + id: execute timeout-minutes: ${{ inputs.timeout }} run: | ${{ inputs.command }} + - name: Release binary + if: ${{ inputs.upload_files != '' }} + uses: softprops/action-gh-release@v1 + with: + files: ${{ inputs.upload_files }} + token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + generate_release_notes: true + tag_name: ${{ steps.execute.outputs.tag_name }} stop-runner: name: Stop self-hosted EC2 runner needs: