Benchmarks #237
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| change-type: | |
| description: How should changes be made after the benchmarks are completed? | |
| required: true | |
| default: ignore | |
| type: choice | |
| options: | |
| - ignore | |
| - commit | |
| - pull_request | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| CARGO_BUILD_JOBS: 128 | |
| TERM: xterm-256color | |
| jobs: | |
| provision-runner: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: AWS Login | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Generate Runner Registration Token | |
| id: get-runner-token | |
| run: | | |
| RESPONSE=$(curl -L -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.RUNNER_TOKEN }}" \ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/${{ github.repository }}/actions/runners/registration-token) | |
| TOKEN=$(echo "$RESPONSE" | jq -r .token) | |
| echo "REGISTER_TOKEN=$TOKEN" >> $GITHUB_ENV | |
| echo "::add-mask::$TOKEN" | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| - name: Terraform Apply | |
| timeout-minutes: 30 | |
| run: | | |
| cd terraform/bench-runner | |
| terraform init | |
| terraform apply -auto-approve -var="registration_token=${{ env.REGISTER_TOKEN }}" -var="github_run_id=${{ github.run_id }}" | |
| - name: Notify on Success | |
| if: success() | |
| uses: appleboy/telegram-action@master | |
| with: | |
| to: ${{ secrets.TELEGRAM_DEVOPS_CHAT }} | |
| token: ${{ secrets.TELEGRAM_DEVOPS_TOKEN }} | |
| format: markdown | |
| disable_web_page_preview: true | |
| message: | | |
| *Status*: ✅ | |
| *Details*: Server provisioned successfully - bench-runner-${{ github.run_id }} | |
| - name: Notify on Failure | |
| if: failure() | |
| uses: appleboy/telegram-action@master | |
| with: | |
| to: ${{ secrets.TELEGRAM_DEVOPS_CHAT }} | |
| token: ${{ secrets.TELEGRAM_DEVOPS_TOKEN }} | |
| format: markdown | |
| disable_web_page_preview: true | |
| message: | | |
| *Status*: 🔥 | |
| *Problem*: Issue with server provisioning - bench-runner-${{ github.run_id }} | |
| *Details*: Check [Benchmark Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) | |
| benchmarks: | |
| needs: provision-runner | |
| runs-on: [bench] | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| env: | |
| RUSTUP_HOME: /tmp/rustup_home | |
| steps: | |
| - name: "ACTIONS: Checkout" | |
| uses: actions/checkout@v4 | |
| - name: "Install: Rust toolchain" | |
| uses: dsherret/rust-toolchain-file@v1 | |
| - name: "Install: Build deps" | |
| run: | | |
| sudo apt update | |
| sudo apt install -y git clang curl libssl-dev llvm libudev-dev cmake wabt protobuf-compiler wget bzip2 | |
| curl -o wasm-opt-linux-x64.tar.gz -L `curl -s https://api.github.com/repos/WebAssembly/binaryen/releases/latest | jq -r '.assets[] | select(.name | contains("x86_64-linux")) | .browser_download_url' |grep -v sha256` | |
| tar xzf wasm-opt-linux-x64.tar.gz && sudo mv binaryen-version_*/bin/* /usr/local/bin/ | |
| wasm-opt --version | |
| - name: print cpu flags | |
| run: | | |
| echo "CPU cores: $(nproc)" | |
| cat /proc/cpuinfo | grep flags | head -n 1 | |
| sleep 1 | |
| - name: Run all benchmarks | |
| run: | | |
| ./scripts/benchmarking/run_all_benchmarks.sh -c vara-dev -m | |
| - name: "ACTIONS: Upload artifact with benchmarking errors (if exist)" | |
| if: ${{ always() }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: benchmarking-errors | |
| path: scripts/benchmarking/benchmarking_errors.txt | |
| if-no-files-found: ignore | |
| - name: "ACTIONS: Upload artifact with updated weights (vara)" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: weights-vara | |
| path: | | |
| scripts/benchmarking/weights-output | |
| - name: "TIP: download the artifacts and use `./scripts/unpack-weights.sh`" | |
| run: | | |
| echo "::notice::You can download the artifacts and use script \`./scripts/unpack-weights.sh\` to apply the changes to your branch" | |
| - name: Create commit | |
| if: ${{ inputs.change-type == 'commit' }} | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: >- | |
| chore(runtime): update weights | |
| file_pattern: "pallets/gear/src/ pallets/gear-builtin/src/ runtime/vara/src/weights/ utils/wasm-instrument/src/gas_metering/" | |
| - name: Create Pull Request | |
| if: ${{ inputs.change-type == 'pull_request' }} | |
| uses: peter-evans/create-pull-request@v6 | |
| with: | |
| add-paths: | | |
| pallets/gear/src/ | |
| pallets/gear-builtin/src/ | |
| runtime/vara/src/weights/ | |
| utils/wasm-instrument/src/gas_metering/ | |
| commit-message: update weights | |
| branch: weights/patch | |
| branch-suffix: short-commit-hash | |
| title: >- | |
| chore(runtime): update weights | |
| body: | | |
| `run_all_benchmarks.sh` script is applied to update weights (branch: `${{ github.ref_name }}`, commit ${{ github.sha }}) | |
| **Note:** If CI fails, try manually updating the heuristic tests: `runtime/vara/src/tests.rs` | |
| labels: | | |
| A0-pleasereview | |
| A4-insubstantial | |
| destroy-runner: | |
| if: always() | |
| needs: [benchmarks] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: AWS Login | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: terraform | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| - name: Terraform Destroy | |
| timeout-minutes: 60 | |
| run: | | |
| cd terraform/bench-runner | |
| terraform init | |
| terraform destroy -auto-approve -var="registration_token=${{ env.REGISTER_TOKEN }}" -var="github_run_id=${{ github.run_id }}" | |
| - name: Notify on Success | |
| if: success() | |
| uses: appleboy/telegram-action@master | |
| with: | |
| to: ${{ secrets.TELEGRAM_DEVOPS_CHAT }} | |
| token: ${{ secrets.TELEGRAM_DEVOPS_TOKEN }} | |
| format: markdown | |
| disable_web_page_preview: true | |
| message: | | |
| *Status*: ✅ | |
| *Details*: Server destroyed successfully - bench-runner-${{ github.run_id }} | |
| - name: Notify on Failure | |
| if: failure() | |
| uses: appleboy/telegram-action@master | |
| with: | |
| to: ${{ secrets.TELEGRAM_DEVOPS_CHAT }} | |
| token: ${{ secrets.TELEGRAM_DEVOPS_TOKEN }} | |
| format: markdown | |
| disable_web_page_preview: true | |
| message: | | |
| *Status*: 🔥 | |
| *Problem*: Issue with server destroying - bench-runner-${{ github.run_id }} | |
| *Details*: Check [Benchmark Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) |