build.yml

name: Build

# Using a single file workflow is the preferred solution for our CI over workflow_runs.
# 1. It generates only 1 action item in the list making it more readable
# 2. It includes the PR/Commit text in the action item
# 3. Artifacts are not available between workflows.

on:
  pull_request:
  push:
    branches:
      - master
      - perm-*
  workflow_dispatch:
    inputs:
      pull_request:
        description: set to pull_request number to execute on external pr
        required: false

env:
  NODE_OPTIONS: "--max-old-space-size=12288 --no-deprecation"
  CARGO_TERM_COLOR: always

jobs:
  ####### Check files and formatting #######
  set-tags:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    outputs:
      git_branch: ${{ steps.check-git-ref.outputs.git_branch }}
      git_ref: ${{ steps.check-git-ref.outputs.git_ref }}
      test_s3_dir: ${{ steps.check-git-ref.outputs.test_s3_dir }}
      image_exists: ${{ steps.check-docker-image.outputs.image_exists }}
      sha: ${{ steps.get-sha.outputs.sha }}
      sha8: ${{ steps.get-sha.outputs.sha8 }}
      polkadot_repo: ${{ steps.get-sha.outputs.polkadot_repo }}
      polkadot_commit: ${{ steps.get-sha.outputs.polkadot_commit }}
      polkadot_ver: ${{ steps.get-sha.outputs.polkadot_ver}}
      latest_rt: ${{ steps.get-sha.outputs.latest_rt }}
      latest_rt_sha8: ${{ steps.get-sha.outputs.latest_rt_sha8 }}
    steps:
      - name: Check git ref
        id: check-git-ref
        # if PR
        # else if manual PR
        # else (push)
        run: |
          if [[ -n "${{ github.event.pull_request.head.sha }}" ]]; then
            echo "git_branch=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_OUTPUT
            echo "git_ref=${{ github.event.pull_request.head.sha }}" >> $GITHUB_OUTPUT
            echo "test_s3_dir=test-pulls/${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT
          elif [[ -n "${{ github.event.inputs.pull_request }}" ]]; then
            echo "git_branch=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_OUTPUT
            echo "test_s3_dir=test-pulls/${{ github.event.inputs.pull_request }}" >> $GITHUB_OUTPUT
            echo "git_ref=refs/pull/${{ github.event.inputs.pull_request }}/head" >> $GITHUB_OUTPUT
          else
            echo "test_s3_dir=test-branches/master" >> $GITHUB_OUTPUT
            echo "git_branch=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
            echo "git_ref=$GITHUB_REF" >> $GITHUB_OUTPUT
          fi
          echo "repo.full_name: ${{ github.event.pull_request.head.repo.full_name }}"
          echo "github.repository: ${{ github.repository }}"

      - uses: actions/checkout@v4
        with:
          ref: ${{ steps.check-git-ref.outputs.git_ref }}
      - name: Get Latest RT Release
        id: get-latest-rt
        run: |
          LATEST_RUNTIME_RELEASE=$(curl -s https://api.github.com/repos/moonbeam-foundation/moonbeam/releases | jq -r '.[] | select(.name | test("runtime";"i")) | .tag_name' | sort -rs | head -n 1 | tr -d '[:blank:]') && [[ ! -z "${LATEST_RUNTIME_RELEASE}" ]]
          echo $LATEST_RUNTIME_RELEASE
          echo "latest_rt=$LATEST_RUNTIME_RELEASE" >> $GITHUB_OUTPUT
      - name: Get Sha
        id: get-sha
        run: |
          echo "sha=$(git log -1 --format='%H')" >> $GITHUB_OUTPUT
          echo "sha8=$(git log -1 --format='%H' | cut -c1-8)" >> $GITHUB_OUTPUT
          echo "polkadot_repo=$(egrep -o 'https.*/polkadot' Cargo.lock | head -1)" >> $GITHUB_OUTPUT
          echo "polkadot_commit=$(egrep -o '/polkadot.*#([^\"]*)' Cargo.lock | \
          head -1 | sed 's/.*#//' |  cut -c1-8)" >> $GITHUB_OUTPUT
          echo "polkadot_ver=$(grep 'frame-system' Cargo.toml | sed -nE 's/.*moonbeam-polkadot-v([0-9]+\.[0-9]+\.[0-9]+).*/\1/p' | head -1)" >> $GITHUB_OUTPUT

          ENDPOINT="https://api.github.com/repos/moonbeam-foundation/moonbeam/git/refs/tags/${{ steps.get-latest-rt.outputs.latest_rt }}"
          RESPONSE=$(curl -s -H "Accept: application/vnd.github.v3+json" $ENDPOINT)
          TYPE=$(echo $RESPONSE | jq -r '.object.type')

          if [[ $TYPE == "commit" ]]
          then
            LATEST_RT_SHA8=$(echo $RESPONSE | jq -r '.object.sha' | cut -c -8)
          elif [[ $TYPE == "tag" ]]
          then
            URL=$(echo $RESPONSE | jq -r '.object.url')
            TAG_RESPONSE=$(curl -s -H "Accept: application/vnd.github.v3+json" $URL)
            TAG_RESPONSE_CLEAN=$(echo $TAG_RESPONSE | tr -d '\000-\037')
            LATEST_RT_SHA8=$(echo $TAG_RESPONSE_CLEAN | jq -r '.object.sha' | cut -c -8)
          fi

          echo $LATEST_RT_SHA8
          echo "latest_rt_sha8=$LATEST_RT_SHA8" >> $GITHUB_OUTPUT

      - name: Check existing docker image
        id: check-docker-image
        run: |
          TAG=sha-${{ steps.get-sha.outputs.sha8 }}
          echo "image_exists=$(docker image inspect moonbeamfoundation/moonbeam:$TAG > /dev/null && echo "true" || echo "false")" >> $GITHUB_OUTPUT
      - name: Display variables
        run: |
          echo git_ref: ${{ steps.check-git-ref.outputs.git_ref }}
          echo sha: ${{ steps.get-sha.outputs.sha }}
          echo sha8: ${{ steps.get-sha.outputs.sha8 }}
          echo image_exists: ${{ steps.check-docker-image.outputs.image_exists }}
          echo latest_rt: ${{ steps.get-latest-rt.outputs.latest_rt }}
          echo latest_rt_sha8: ${{ steps.get-sha.outputs.latest_rt_sha8 }}
          echo polkadot_repo: ${{ steps.get-sha.outputs.polkadot_repo }}
          echo polkadot_commit: ${{ steps.get-sha.outputs.polkadot_commit }}
          echo polkadot_ver: ${{ steps.get-sha.outputs.polkadot_ver }}

  check-copyright:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    needs: ["set-tags"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - name: Find un-copyrighted files
        run: |
          find . \! -name '*.expanded.rs' -name '*.rs' -exec grep  -H -E -o -c Copyright {} \; | grep ':0' || true
          FILECOUNT=$(find . \! -name '*.expanded.rs' -name '*.rs' -exec grep  -H -E -o -c Copyright {} \; | grep -c ':0' || true)
          if [[ $FILECOUNT -eq 0 ]]; then
            true
          else
            false
          fi

  check-links:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    needs: ["set-tags"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - uses: gaurav-nelson/github-action-markdown-link-check@v1
        with:
          use-quiet-mode: "yes"
          max-depth: 4

  check-biome:
    name: "Check with Biome"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    needs: ["set-tags"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - uses: pnpm/action-setup@v4
        with:
          version: 9
      - name: Setup Node.js
        uses: actions/setup-node@v4
        with:
          node-version-file: "test/.nvmrc"
          cache: "pnpm"
          cache-dependency-path: pnpm-lock.yaml
      - run: pnpm install
      - run: pnpm check

  check-cargo-toml-format:
    name: "Check Cargo.toml files format"
    runs-on:
      labels: ubuntu-latest
    permissions:
      contents: read
    needs: ["set-tags"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      # With rustup's nice new toml format, we just need to run rustup show to install the toolchain
      # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659
      - name: Setup Rust toolchain
        run: |
          rustup override unset
          rustup show
      - name: Check Cargo.toml files format with toml_sort
        run: ./scripts/check-cargo-toml-files-format.sh

  check-forbid-evm-reentrancy:
    name: "Check 'forbid-evm-reentrancy'"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    needs: ["set-tags"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - name: Setup Rust toolchain
        run: |
          rustup override unset
          rustup show
      - name: Verifies all 'pallet-evm/ethereum' use 'forbid-evm-reentrancy' feature
        run: ./scripts/check-forbid-evm-reentrancy.sh

  check-rust-fmt:
    name: "Check with rustfmt"
    runs-on: ubuntu-latest
    permissions:
      contents: read
    needs: ["set-tags"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      # With rustup's nice new toml format, we just need to run rustup show to install the toolchain
      # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659
      - name: Setup Rust toolchain
        run: |
          rustup override unset
          rustup show
      - name: Format code with rustfmt
        run: cargo fmt -- --check

  ####### Building and Testing binaries #######

  cargo-clippy:
    runs-on:
      labels: bare-metal
    permissions:
      contents: read
    needs: ["set-tags"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
      - name: Setup Rust toolchain
        run: |
          rustup override unset
          rustup show
      # Development branch clippy check
      - name: Clippy (dev)
        if: github.ref != 'refs/heads/master' && !startsWith(github.ref, 'perm-')
        run: SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo clippy --features try-runtime,runtime-benchmarks
      # Main branch (master, perm-*) clippy check
      # Disallows: todo
      - name: Clippy (main)
        if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'perm-')
        run: SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo clippy --features try-runtime,runtime-benchmarks -- -Dclippy::todo

  build:
    runs-on:
      labels: bare-metal
    permissions:
      contents: read
    needs: ["set-tags"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - name: Cargo build
        uses: ./.github/workflow-templates/cargo-build
        with:
          features: metadata-hash
      - name: Upload runtimes
        uses: actions/upload-artifact@v4
        with:
          name: runtimes
          path: runtimes
      - name: Upload uncompressed runtimes
        uses: actions/upload-artifact@v4
        with:
          name: uncompressed-runtimes
          path: uncompressed-runtimes
      - name: Upload binary
        uses: actions/upload-artifact@v4
        with:
          name: moonbeam
          path: build

  check-wasm-size:
    name: "Check WASM runtimes with Twiggy"
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
    if: ${{ github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork }}
    needs: ["set-tags", "build"]
    env:
      GH_TOKEN: ${{ github.token }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      # With rustup's nice new toml format, we just need to run rustup show to install the toolchain
      # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659
      - name: Setup Rust toolchain
        run: |
          rustup override unset
          rustup show
      - name: Download Twiggy
        run: cargo install twiggy
      - name: Lookup for latest target branch build
        run: |
          TARGET_BRANCH=${{ github.event.pull_request.base.ref }}
          LATEST_TARGET_BRANCH_BUILD=$(gh run -R moonbeam-foundation/moonbeam list -w Build --limit=100 --json databaseId,url,headBranch,event,status,conclusion,createdAt --jq ".[] | select(.headBranch == \"$TARGET_BRANCH\" and .event == \"push\" and .status == \"completed\" and .conclusion == \"success\") | .databaseId" | head -n 1)
          echo "LATEST_TARGET_BRANCH_BUILD=$LATEST_TARGET_BRANCH_BUILD" >> $GITHUB_OUTPUT
      - name: Download latest target branch build artifacts
        run: |
          gh run download $LATEST_TARGET_BRANCH_BUILD -n uncompressed-runtimes --dir uncompressed-runtimes-target-branch
          gh run download $LATEST_TARGET_BRANCH_BUILD -n runtimes --dir runtimes-target-branch
      - name: Check Runtimes size for target branch
        run: |
          PREVIOUS_MOONBASE=$(du -k runtimes-target-branch/* | awk '/moonbase_runtime/ {print $1}')
          PREVIOUS_MOONBEAM=$(du -k runtimes-target-branch/* | awk '/moonbeam_runtime/ {print $1}')
          PREVIOUS_MOONRIVER=$(du -k runtimes-target-branch/* | awk '/moonriver_runtime/ {print $1}')
          echo "PREVIOUS_MOONBASE=$PREVIOUS_MOONBASE" >> $GITHUB_ENV
          echo "PREVIOUS_MOONBEAM=$PREVIOUS_MOONBEAM" >> $GITHUB_ENV
          echo "PREVIOUS_MOONRIVER=$PREVIOUS_MOONRIVER" >> $GITHUB_ENV
      - name: "Download branch built runtime"
        uses: actions/download-artifact@v4
        with:
          name: runtimes
          path: runtimes-current-branch
      - name: "Download branch built uncompressed-runtimes"
        uses: actions/download-artifact@v4
        with:
          name: uncompressed-runtimes
          path: uncompressed-runtimes-current-branch
      - name: Check Runtimes size for current branch
        run: |
          CURRENT_MOONBASE=$(du -k runtimes-current-branch/* | awk '/moonbase_runtime/ {print $1}')
          CURRENT_MOONBEAM=$(du -k runtimes-current-branch/* | awk '/moonbeam_runtime/ {print $1}')
          CURRENT_MOONRIVER=$(du -k runtimes-current-branch/* | awk '/moonriver_runtime/ {print $1}')
          echo "CURRENT_MOONBASE=$CURRENT_MOONBASE" >> $GITHUB_ENV
          echo "CURRENT_MOONBEAM=$CURRENT_MOONBEAM" >> $GITHUB_ENV
          echo "CURRENT_MOONRIVER=$CURRENT_MOONRIVER" >> $GITHUB_ENV
      - name: Fetch latest release tag
        id: fetch_latest_release
        run: |
          LATEST_RELEASE_TAG=$(gh api repos/moonbeam-foundation/moonbeam/releases --paginate --jq '.[] | select(.tag_name | test("^runtime-\\d+$")) | .tag_name' | sort -V | tail -n 1)
          echo "LATEST_RELEASE_TAG=$LATEST_RELEASE_TAG" >> $GITHUB_ENV
          echo $LATEST_RELEASE_TAG
      - name: Download latest release runtimes
        run: |
          gh release download $LATEST_RELEASE_TAG -R moonbeam-foundation/moonbeam -p "moonbeam-runtime-${LATEST_RELEASE_TAG#runtime-}.wasm" -p "moonbase-runtime-${LATEST_RELEASE_TAG#runtime-}.wasm" -p "moonriver-runtime-${LATEST_RELEASE_TAG#runtime-}.wasm" --dir runtimes-latest-release
      - name: Check Runtimes size for latest release
        run: |
          LATEST_MOONBASE=$(du -k runtimes-latest-release/* | awk '/moonbase-runtime/ {print $1}')
          LATEST_MOONBEAM=$(du -k runtimes-latest-release/* | awk '/moonbeam-runtime/ {print $1}')
          LATEST_MOONRIVER=$(du -k runtimes-latest-release/* | awk '/moonriver-runtime/ {print $1}')
          echo "LATEST_MOONBASE=$LATEST_MOONBASE" >> $GITHUB_ENV
          echo "LATEST_MOONBEAM=$LATEST_MOONBEAM" >> $GITHUB_ENV
          echo "LATEST_MOONRIVER=$LATEST_MOONRIVER" >> $GITHUB_ENV
      - name: Create Twiggy diff reports
        run: |
          # Install Twiggy if not already installed
          if ! command -v twiggy &> /dev/null; then
            cargo install twiggy
          fi

          # Generate Twiggy diff reports in JSON format with top 100 entries
          mkdir -p twiggy-diff-reports
          twiggy diff -n 100 --format json uncompressed-runtimes-target-branch/moonbase_runtime.wasm uncompressed-runtimes-current-branch/moonbase_runtime.wasm > twiggy-diff-reports/twiggy_diff_moonbase.json
          twiggy diff -n 100 --format json uncompressed-runtimes-target-branch/moonbeam_runtime.wasm uncompressed-runtimes-current-branch/moonbeam_runtime.wasm > twiggy-diff-reports/twiggy_diff_moonbeam.json
          twiggy diff -n 100 --format json uncompressed-runtimes-target-branch/moonriver_runtime.wasm uncompressed-runtimes-current-branch/moonriver_runtime.wasm > twiggy-diff-reports/twiggy_diff_moonriver.json

      - name: Upload Twiggy diff reports
        uses: actions/upload-artifact@v4
        with:
          name: twiggy-diff-reports
          path: twiggy-diff-reports
      - name: Compare Runtimes sizes
        run: |
          # Create or truncate the file
          echo "" > runtime_size_report.md

          MOONBASE_DIFF=$((CURRENT_MOONBASE - PREVIOUS_MOONBASE))
          MOONBEAM_DIFF=$((CURRENT_MOONBEAM - PREVIOUS_MOONBEAM))
          MOONRIVER_DIFF=$((CURRENT_MOONRIVER - PREVIOUS_MOONRIVER))

          LATEST_MOONBASE_DIFF=$((CURRENT_MOONBASE - LATEST_MOONBASE))
          LATEST_MOONBEAM_DIFF=$((CURRENT_MOONBEAM - LATEST_MOONBEAM))
          LATEST_MOONRIVER_DIFF=$((CURRENT_MOONRIVER - LATEST_MOONRIVER))

          get_status_emoji() {
            local size=$1
            local diff=$2
            if [ $size -gt 2400 ]; then
              echo "🚨"
            elif [ $diff -gt 0 ]; then
              echo "⚠️"
            else
              echo "✅"
            fi
          }

          MOONBASE_STATUS=$(get_status_emoji $CURRENT_MOONBASE $MOONBASE_DIFF)
          MOONBEAM_STATUS=$(get_status_emoji $CURRENT_MOONBEAM $MOONBEAM_DIFF)
          MOONRIVER_STATUS=$(get_status_emoji $CURRENT_MOONRIVER $MOONRIVER_DIFF)

          LATEST_MOONBASE_STATUS=$(get_status_emoji $CURRENT_MOONBASE $LATEST_MOONBASE_DIFF)
          LATEST_MOONBEAM_STATUS=$(get_status_emoji $CURRENT_MOONBEAM $LATEST_MOONBEAM_DIFF)
          LATEST_MOONRIVER_STATUS=$(get_status_emoji $CURRENT_MOONRIVER $LATEST_MOONRIVER_DIFF)

          MOONBASE_MSG="Moonbase runtime: ${CURRENT_MOONBASE} KB ($( [ $MOONBASE_DIFF -eq 0 ] && echo "no changes" || echo "$( [ $MOONBASE_DIFF -gt 0 ] && echo "+" )${MOONBASE_DIFF} KB")) ${MOONBASE_STATUS}"
          MOONBEAM_MSG="Moonbeam runtime: ${CURRENT_MOONBEAM} KB ($( [ $MOONBEAM_DIFF -eq 0 ] && echo "no changes" || echo "$( [ $MOONBEAM_DIFF -gt 0 ] && echo "+" )${MOONBEAM_DIFF} KB")) ${MOONBEAM_STATUS}"
          MOONRIVER_MSG="Moonriver runtime: ${CURRENT_MOONRIVER} KB ($( [ $MOONRIVER_DIFF -eq 0 ] && echo "no changes" || echo "$( [ $MOONRIVER_DIFF -gt 0 ] && echo "+" )${MOONRIVER_DIFF} KB")) ${MOONRIVER_STATUS}"

          LATEST_MOONBASE_MSG="Moonbase runtime: ${CURRENT_MOONBASE} KB ($( [ $LATEST_MOONBASE_DIFF -eq 0 ] && echo "no changes" || echo "$( [ $LATEST_MOONBASE_DIFF -gt 0 ] && echo "+" )${LATEST_MOONBASE_DIFF} KB compared to latest release")) ${LATEST_MOONBASE_STATUS}"
          LATEST_MOONBEAM_MSG="Moonbeam runtime: ${CURRENT_MOONBEAM} KB ($( [ $LATEST_MOONBEAM_DIFF -eq 0 ] && echo "no changes" || echo "$( [ $LATEST_MOONBEAM_DIFF -gt 0 ] && echo "+" )${LATEST_MOONBEAM_DIFF} KB compared to latest release")) ${LATEST_MOONBEAM_STATUS}"
          LATEST_MOONRIVER_MSG="Moonriver runtime: ${CURRENT_MOONRIVER} KB ($( [ $LATEST_MOONRIVER_DIFF -eq 0 ] && echo "no changes" || echo "$( [ $LATEST_MOONRIVER_DIFF -gt 0 ] && echo "+" )${LATEST_MOONRIVER_DIFF} KB compared to latest release")) ${LATEST_MOONRIVER_STATUS}"

          echo "### WASM runtime size check:" > runtime_size_report.md
          echo "" >> runtime_size_report.md
          echo "#### Compared to target branch" >> runtime_size_report.md
          echo "" >> runtime_size_report.md
          echo "$MOONBASE_MSG" >> runtime_size_report.md
          echo "" >> runtime_size_report.md
          echo "$MOONBEAM_MSG" >> runtime_size_report.md
          echo "" >> runtime_size_report.md
          echo "$MOONRIVER_MSG" >> runtime_size_report.md
          echo "" >> runtime_size_report.md
          echo "#### Compared to latest release (${LATEST_RELEASE_TAG})" >> runtime_size_report.md
          echo "" >> runtime_size_report.md
          echo "$LATEST_MOONBASE_MSG" >> runtime_size_report.md
          echo "" >> runtime_size_report.md
          echo "$LATEST_MOONBEAM_MSG" >> runtime_size_report.md
          echo "" >> runtime_size_report.md
          echo "$LATEST_MOONRIVER_MSG" >> runtime_size_report.md
          cat runtime_size_report.md
      - name: Find Comment
        uses: peter-evans/find-comment@v3
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: "github-actions[bot]"
          body-includes: "WASM runtime size check"
      - name: Create or update comment
        uses: peter-evans/create-or-update-comment@v4
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body-path: runtime_size_report.md
          edit-mode: replace

  rust-test:
    runs-on:
      labels: bare-metal
    permissions:
      contents: read
    needs: ["set-tags"]
    env:
      RUSTC_WRAPPER: "sccache"
      CARGO_INCREMENTAL: "0"
      SCCACHE_CACHE_SIZE: "100GB"
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - name: Run sccache-cache
        uses: mozilla-actions/sccache-action@v0.0.7
      - name: Setup Variables
        shell: bash
        run: |
          echo "RUSTFLAGS=-C opt-level=3 -D warnings -C linker=clang -C link-arg=-fuse-ld=$(pwd)/mold/bin/mold" >> $GITHUB_ENV
      - name: Setup Mold Linker
        shell: bash
        run: |
          mkdir -p mold
          curl -L --retry 10 --silent --show-error https://github.com/rui314/mold/releases/download/v2.30.0/mold-2.30.0-$(uname -m)-linux.tar.gz | tar -C $(realpath mold) --strip-components=1 -xzf -
      # With rustup's nice new toml format, we just need to run rustup show to install the toolchain
      # https://github.com/actions-rs/toolchain/issues/126#issuecomment-782989659
      - name: Setup Rust toolchain
        run: |
          if ! which "rustup" > /dev/null; then
            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
          fi
          rustup override unset
          rustup show
          rustup target add wasm32-unknown-unknown
      # Checks are run after uploading artifacts since they are modified by the tests
      - name: Unit tests
        run: |
          cargo test --profile testnet --workspace --features=evm-tracing,runtime-benchmarks
      - name: Run sccache stat for check pre test
        run: ${SCCACHE_PATH} --show-stats

  # Renable typegen_check when we have a bot in place to update PR for us in case of missing it

  # typegen_check:
  #   needs: ["set-tags", "build"]
  #   name: "Check Rust/TS bindings are up to date"
  #   runs-on: ubuntu-latest
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v4
  #       with:
  #         ref: ${{ needs.set-tags.outputs.git_ref }}
  #     - uses: pnpm/action-setup@v4
  #       with:
  #         version: 9
  #     - uses: actions/setup-node@v4
  #       with:
  #         node-version-file: "test/.nvmrc"
  #         cache: "pnpm"
  #         cache-dependency-path: pnpm-lock.yaml
  #     - name: "Download branch built node"
  #       uses: actions/download-artifact@v4
  #       with:
  #         name: moonbeam
  #         path: target/release
  #     - run: chmod uog+x target/release/moonbeam
  #     - name: Run Typegen
  #       run: |
  #         pnpm i

  #         cd test
  #         pnpm typegen
  #     - name: Check for changes
  #       run: |
  #         cd typescript-api
  #         if [ -n "$(git status --porcelain .)" ]; then
  #           echo "Typegen produced changes. Please run 'pnpm typegen' locally and commit the changes."
  #           false
  #         else
  #           echo "No changes"
  #           true
  #         fi

  dev-test:
    runs-on:
      labels: bare-metal
    permissions:
      contents: read
    needs: ["set-tags", "build"]
    timeout-minutes: 20
    strategy:
      fail-fast: false
      matrix:
        chain: ["moonbase", "moonriver", "moonbeam"]
        shard: [1, 2, 3, 4]
    env:
      GH_WORKFLOW_MATRIX_CHAIN: ${{ matrix.chain }}
      DEBUG_COLORS: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - name: Create local folders
        run: |
          mkdir -p target/release/wbuild/${{ matrix.chain }}-runtime/
      - name: "Download branch built runtime"
        uses: actions/download-artifact@v4
        with:
          name: runtimes
          path: target/release/wbuild/${{ matrix.chain }}-runtime/
      - name: "Download branch built node"
        uses: actions/download-artifact@v4
        with:
          name: moonbeam
          path: target/release
      - name: "Run Moonwall Dev Tests"
        uses: ./.github/workflow-templates/dev-tests
        with:
          moonwall_environment: dev_${{ matrix.chain }}
          shard: ${{ matrix.shard }}
          shard-count: 4
      - name: Upload HTML report to s3
        if: ${{ !github.event.pull_request.head.repo.fork}}
        uses: opslayertech/upload-s3-action@v1.1.0
        id: S3
        with:
          aws_key_id: ${{ secrets.S3_COVERAGE_ID }}
          aws_secret_access_key: ${{ secrets.S3_COVERAGE_KEY }}
          aws_bucket: ${{ vars.S3_COVERAGE_BUCKET }}
          destination_dir: "${{ needs.set-tags.outputs.test_s3_dir }}"
          source_dir: "test/html"
          acl: "none"
      # - name: Upload to Moonscope
      #   if: github.event.pull_request.head.repo.full_name == github.repository
      #   run: |
      #     curl --location 'https://api.moonscope.kaki.dev/insert' \
      #     -X POST \
      #     --header 'moonwallenv: dev_${{ matrix.chain }}' \
      #     --header 'Authorization: Bearer ${{ secrets.MOONSCOPE_TOKEN }}' \
      #     --header 'table: dev_reports' \
      #     --header 'branch: ${{ needs.set-tags.outputs.git_branch }}' \
      #     --header 'Content-Type: application/json' \
      #     --max-time 30 \
      #     -d@test/tmp/testResults.json || echo "Curl command failed but continuing"

  typescript-tracing-tests:
    if: >
      (github.event_name == 'pull_request' && !github.event.pull_request.head.repo.fork) || 
      (github.event_name == 'push' && github.ref == 'refs/heads/master')
    runs-on:
      labels: bare-metal
    permissions:
      contents: read
    needs: ["set-tags", "build", "dev-test"]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - uses: actions/download-artifact@v4
        with:
          name: moonbeam
          path: build
      - name: Use pnpm
        uses: pnpm/action-setup@v4
        with:
          version: 9
      - name: Use Node.js
        uses: actions/setup-node@v4
        with:
          node-version-file: "test/.nvmrc"
          cache: "pnpm"
          cache-dependency-path: pnpm-lock.yaml
      - run: |
          mkdir -p target/release
      - name: "Download branch built node"
        uses: actions/download-artifact@v4
        with:
          name: moonbeam
          path: target/release
      - name: Get tracing runtimes
        run: |
          ./scripts/build-last-tracing-runtime.sh ${{ needs.set-tags.outputs.git_branch }}
          mkdir -p test/moonbase-overrides/
          mv build/wasm/moonbase-runtime-local-substitute-tracing.wasm test/moonbase-overrides/
      - name: Preparing the repository
        run: |
          chmod uog+x build/moonbeam
          chmod uog+x target/release/moonbeam
      - name: Running Tracing Tests
        env:
          DEBUG_COLOURS: "1"
          NODE_OPTIONS: "--max-old-space-size=12288"
        run: |
          cd test
          pnpm install
          pnpm compile-solidity
          pnpm moonwall test dev_moonbase_tracing

  docker-moonbeam:
    runs-on: ubuntu-latest
    permissions:
      contents: read
    needs: ["set-tags", "build"]
    if: ${{ needs.set-tags.outputs.image_exists == 'false' && !github.event.pull_request.head.repo.fork }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - uses: actions/download-artifact@v4
        with:
          name: moonbeam
          path: build
      - name: Prepare
        id: prep
        run: |
          DOCKER_IMAGE=moonbeamfoundation/moonbeam
          TAGS="${DOCKER_IMAGE}:sha-${{ needs.set-tags.outputs.sha8 }}"
          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
      - name: Cargo build
        uses: ./.github/workflow-templates/publish-docker
        with:
          dockerhub_username: ${{ secrets.MBF_DOCKERHUB_USERNAME }}
          dockerhub_password: ${{ secrets.MBF_DOCKERHUB_PASSWORD }}
          image_tags: ${{ steps.prep.outputs.tags }}
          image_title: ${{ github.event.repository.name }}
          image_description: ${{ github.event.repository.description }}
          image_url: ${{ github.event.repository.html_url }}
          image_source: ${{ github.event.repository.clone_url }}
          image_created: ${{ steps.prep.outputs.created }}
          image_revision: ${{ github.sha }}
          image_licenses: ${{ github.event.repository.license.spdx_id }}


  lazy-loading-tests:
    runs-on:
      labels: bare-metal
    permissions:
      contents: read
    needs: ["set-tags", "build"]
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        chain: ["moonbeam"]
    env:
      GH_WORKFLOW_MATRIX_CHAIN: ${{ matrix.chain }}
      DEBUG_COLORS: 1
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - uses: pnpm/action-setup@v4
        with:
          version: 9
      - uses: actions/setup-node@v4
        with:
          node-version-file: "test/.nvmrc"
      - name: Create local folders
        run: |
          mkdir -p target/release/wbuild/${{ matrix.chain }}-runtime/
          mkdir -p test/tmp
      - name: "Download branch built runtime"
        uses: actions/download-artifact@v4
        with:
          name: runtimes
          path: target/release/wbuild/${{ matrix.chain }}-runtime/
      - name: "Download branch built node"
        uses: actions/download-artifact@v4
        with:
          name: moonbeam
          path: target/release
      - name: "Run lazy loading tests"
        run: |
          chmod uog+x target/release/moonbeam
          cd test
          pnpm install
          pnpm moonwall test lazy_loading_${{ matrix.chain }}
      - name: Zip and Upload Node Logs on Failure
        if: failure()
        run: |
          TIMESTAMP=$(date +%Y%m%d%H%M%S)
          export NODE_LOGS_ZIP="node_logs_$TIMESTAMP.zip"
          MOST_RECENT_ZOMBIE_DIR=$(ls -td /tmp/zombie-* | head -n 1)
          find $MOST_RECENT_ZOMBIE_DIR -maxdepth 1 -type f -name '*.log' -exec zip -r $NODE_LOGS_ZIP {} \;
          echo "NODE_LOGS_ZIP=${NODE_LOGS_ZIP}" >> $GITHUB_ENV
      - uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: failed-node-logs
          path: ${{ env.NODE_LOGS_ZIP }}

  chopsticks-upgrade-test:
    runs-on:
      labels: bare-metal
    needs: ["set-tags", "build"]
    strategy:
      fail-fast: false
      matrix:
        chain: ["moonbase", "moonriver", "moonbeam"]
    env:
      GH_WORKFLOW_MATRIX_CHAIN: ${{ matrix.chain }}
      DEBUG_COLORS: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - uses: pnpm/action-setup@v4
        with:
          version: 9
      - uses: actions/setup-node@v4
        with:
          node-version-file: "test/.nvmrc"
          cache: "pnpm"
          cache-dependency-path: pnpm-lock.yaml
      - name: Create local folders
        run: |
          mkdir -p target/release/wbuild/${{ matrix.chain }}-runtime/
          mkdir -p test/tmp/node_logs
      - name: "Download runtime"
        uses: actions/download-artifact@v4
        with:
          name: runtimes
          path: target/release/wbuild/${{ matrix.chain }}-runtime/
      - name: "Install and run upgrade test"
        run: |
          cd test
          pnpm install
      - name: Run Upgrade Test (with retry)
        uses: nick-fields/retry@v3
        with:
          max_attempts: 3
          timeout_minutes: 2
          retry_on: error
          command: |
            cd test
            pnpm moonwall test upgrade_${{matrix.chain}}

  zombie_upgrade_test:
    runs-on:
      labels: bare-metal
    needs: ["set-tags", "build"]
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        ## TODO: add moonriver here when it is ready
        chain: ["moonbase", "moonbeam"]
    env:
      GH_WORKFLOW_MATRIX_CHAIN: ${{ matrix.chain }}
      DEBUG_COLORS: 1
      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ needs.set-tags.outputs.git_ref }}
      - uses: pnpm/action-setup@v4
        with:
          version: 9
      - uses: actions/setup-node@v4
        with:
          node-version-file: "test/.nvmrc"
      - name: Create local folders
        run: |
          mkdir -p target/release/wbuild/${{ matrix.chain }}-runtime/
          mkdir -p test/tmp
      - name: "Download branch built runtime"
        uses: actions/download-artifact@v4
        with:
          name: runtimes
          path: target/release/wbuild/${{ matrix.chain }}-runtime/
      - name: "Download branch built node"
        uses: actions/download-artifact@v4
        with:
          name: moonbeam
          path: target/release
      - name: Retrieve moonbeam binary from docker (for plainSpec generation)
        run: |
          MOONBEAM_COMMIT=${{ needs.set-tags.outputs.latest_rt_sha8 }}
          DOCKER_TAG="moonbeamfoundation/moonbeam:sha-$MOONBEAM_COMMIT"

          docker rm -f moonbeam_container 2> /dev/null | true
          docker create --name moonbeam_container $DOCKER_TAG bash
          docker cp moonbeam_container:moonbeam/moonbeam test/tmp/moonbeam_rt
          docker rm -f moonbeam_container
      - name: Prepare Chainspecs
        run: |
          cd test
          pnpm install

          ## Generate old spec using latest published node, modify it, and generate raw spec
          chmod uog+x tmp/moonbeam_rt
          chmod uog+x ../target/release/moonbeam
          tmp/moonbeam_rt build-spec --chain ${{ matrix.chain }}-local > tmp/${{ matrix.chain }}-plain-spec.json
          pnpm tsx scripts/modify-plain-specs.ts process tmp/${{ matrix.chain }}-plain-spec.json tmp/${{ matrix.chain }}-modified-spec.json
          tmp/moonbeam_rt build-spec --chain tmp/${{ matrix.chain }}-modified-spec.json --raw > tmp/${{ matrix.chain }}-raw-spec.json
          pnpm tsx scripts/preapprove-rt-rawspec.ts process tmp/${{ matrix.chain }}-raw-spec.json tmp/${{ matrix.chain }}-modified-raw-spec.json ../target/release/wbuild/${{ matrix.chain }}-runtime/${{ matrix.chain }}_runtime.compact.compressed.wasm
      - name: "Run zombie upgrade test"
        run: |
          cd test
          pnpm moonwall test zombie_${{ matrix.chain }}
      - name: "Run zombie RPC test"
        run: |
          cd test
          pnpm moonwall test zombie_${{ matrix.chain }}_rpc
      - name: Zip and Upload Node Logs on Failure
        if: failure()
        run: |
          TIMESTAMP=$(date +%Y%m%d%H%M%S)
          export NODE_LOGS_ZIP="node_logs_$TIMESTAMP.zip"
          MOST_RECENT_ZOMBIE_DIR=$(ls -td /tmp/zombie-* | head -n 1)
          find $MOST_RECENT_ZOMBIE_DIR -maxdepth 1 -type f -name '*.log' -exec zip -r $NODE_LOGS_ZIP {} \;
          echo "NODE_LOGS_ZIP=${NODE_LOGS_ZIP}" >> $GITHUB_ENV
      - uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: failed-node-logs
          path: ${{ env.NODE_LOGS_ZIP }}