Feature/issue 231 binary release workflow (#270)

update the workflow for release
timeplus-io · Nov 14, 2023 · c5e8aaa · c5e8aaa
1 parent 1391741
commit c5e8aaa
Show file tree

Hide file tree

Showing 2 changed files with 158 additions and 5 deletions.
diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml
@@ -5,13 +5,14 @@ on: # yamllint disable-line rule:truthy
   workflow_dispatch:
 
 jobs:
-  Build:
+  Build_Linux_X86_64:
     uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
     with:
       ec2-instance-type: c5.4xlarge
       ec2-image-id: ami-042a37e33a285c22b
       submodules: 'recursive'
       run_mode: 'start' # start ec2 on demand instance
+      upload_files: proton-*-Linux-x86_64
       command: |
         cd $GITHUB_WORKSPACE
 
@@ -29,13 +30,21 @@ jobs:
         rm cache.tar.gz
 
         # compiling
-        ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
+        ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
         
         if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
           echo "Compiling proton Failed"
           exit 127
         fi
 
+        # get proton tag
+        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
+        echo "Proton tag: $PROTON_TAG"
+        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
+
+        PROTON_BINARY=proton-$PROTON_TAG-Linux-x86_64
+        echo "Proton Binary Name: $PROTON_BINARY"
+
         docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
         cd $GITHUB_WORKSPACE/docker/server
 
@@ -48,6 +57,8 @@ jobs:
         # upload build cache
         tar -zcf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache .
         aws s3 cp --no-progress ./cache.tar.gz s3://tp-internal/proton/
+        
+        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -56,13 +67,14 @@ jobs:
       GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-  BuildArm:
+  Build_Linux_Arm64:
     uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
     with:
       ec2-instance-type: c6g.4xlarge
       ec2-image-id: ami-0f3dbc4cc9994fdee
       submodules: 'recursive'
       run_mode: 'start' # start ec2 on demand instance
+      upload_files: proton-*-Linux-aarch64
       command: |
         cd $GITHUB_WORKSPACE
 
@@ -80,7 +92,16 @@ jobs:
         rm cache-arm.tar.gz
 
         # compiling
-        ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
+        ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output
+        
+        # get proton tag
+        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
+        echo "Proton tag: $PROTON_TAG"
+        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
+
+        PROTON_BINARY=proton-$PROTON_TAG-Linux-aarch64
+        echo "Proton Binary Name: $PROTON_BINARY"
+        
         docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
         cd $GITHUB_WORKSPACE/docker/server
 
@@ -93,6 +114,8 @@ jobs:
         # upload build cache
         tar -zcf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache .
         aws s3 cp --no-progress ./cache-arm.tar.gz s3://tp-internal/proton/
+        
+        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
     secrets:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -102,7 +125,7 @@ jobs:
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
   CommitTag:
-    needs: [Build, BuildArm]
+    needs: [Build_Linux_X86_64, Build_Linux_Arm64]
     uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
     with:
       ec2-instance-type: c5.4xlarge
@@ -154,3 +177,120 @@ jobs:
       GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
       DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+  Build_Darwin_X86_64:
+    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
+    with:
+      ec2-instance-type: c5.4xlarge
+      ec2-image-id: ami-042a37e33a285c22b
+      submodules: 'recursive'
+      run_mode: 'start' # start ec2 on demand instance
+      upload_files: proton-*-Darwin-x86_64
+      command: |
+        cd $GITHUB_WORKSPACE
+
+        # git config
+        git config user.name "proton-robot"
+        git config user.email "[email protected]"
+        
+        # update version
+        ./release --version patch
+
+        # download the pre-built binary of v8 (this is only for cross-compile)
+        aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-x64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/
+        tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/
+        rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque
+        
+        # prepare build cache
+        mkdir $GITHUB_WORKSPACE/ccache
+        echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
+
+        # compiling
+        ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin
+
+        if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
+          echo "Compiling proton Failed"
+          exit 127
+        fi
+
+        # clean ccache and build footprint
+        rm -rf $GITHUB_WORKSPACE/ccache
+        rm -rf $GITHUB_WORKSPACE/build_docker
+
+        # get proton tag
+        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
+        echo "Proton tag: $PROTON_TAG"
+        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
+
+        PROTON_BINARY=proton-$PROTON_TAG-Darwin-x86_64
+        echo "Proton Binary Name: $PROTON_BINARY"
+
+        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+  Build_Darwin_Arm64:
+    uses: timeplus-io/proton/.github/workflows/run_command.yml@develop
+    with:
+      ec2-instance-type: c6g.4xlarge
+      ec2-image-id: ami-0f3dbc4cc9994fdee
+      submodules: 'recursive'
+      run_mode: 'start' # start ec2 on demand instance
+      upload_files: proton-*-Darwin-arm64
+      command: |
+        cd $GITHUB_WORKSPACE
+
+        # git config
+        git config user.name "proton-robot"
+        git config user.email "[email protected]"
+        
+        # update version
+        ./release --version patch
+
+        # download the pre-built binary of v8 (this is only for cross-compile)
+        aws s3 cp --no-progress s3://tp-internal/proton/cross-compile-prebuilt-binary/v8-cmake-arm64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/
+        tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/
+        rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot
+        chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque
+
+        # prepare build cache
+        mkdir $GITHUB_WORKSPACE/ccache
+        echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf
+
+        # compiling
+        ./docker/packager/packager --package-type binary --docker-image-version clang-16 --build-type release --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-16-darwin-aarch64
+        if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then
+          echo "Compiling proton Failed"
+          exit 127
+        fi
+
+        # clean ccache and build footprint
+        rm -rf $GITHUB_WORKSPACE/ccache
+        rm -rf $GITHUB_WORKSPACE/build_docker
+
+        # get proton tag
+        PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'`
+        echo "Proton tag: $PROTON_TAG"
+        echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT
+
+        PROTON_BINARY=proton-$PROTON_TAG-Darwin-arm64
+        echo "Proton Binary Name: $PROTON_BINARY"
+
+        mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY
+    secrets:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_REGION: ${{ secrets.AWS_REGION }}
+      AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
+      GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/run_command.yml b/.github/workflows/run_command.yml
@@ -53,6 +53,10 @@ on:
         required: false
         type: string
         default: 'start_spot' # start_spot, start
+      upload_files:
+        required: false
+        type: string
+        default: ''
     secrets:
       AWS_ACCESS_KEY_ID:
         required: true
@@ -152,9 +156,18 @@ jobs:
       - name: Prepare directory
         run: mkdir /test_output /artifacts
       - name: execute
+        id: execute
         timeout-minutes: ${{ inputs.timeout }}
         run: |
           ${{ inputs.command }}
+      - name: Release binary
+        if: ${{ inputs.upload_files != '' }}
+        uses: softprops/action-gh-release@v1
+        with:
+          files: ${{ inputs.upload_files }}
+          token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+          generate_release_notes: true
+          tag_name: ${{ steps.execute.outputs.tag_name }}
   stop-runner:
     name: Stop self-hosted EC2 runner
     needs: