ci-pipeline-branch.yml

name: CI PIPELINE - BRANCH

on:
  pull_request:
    types:
      - opened
      - synchronize

    branches:
      - 'branch*'

concurrency:
  group: ${{ github.event.number }}
  cancel-in-progress: true


jobs:

  basic-checker:
    runs-on: [ self-hosted, normal ]
    name: RUN CHECKER
    env:
      PR_NUMBER: ${{ github.event.number }}
      GH_TOKEN: ${{ github.token }}
    outputs:
      PASS: ${{ steps.check.outputs.pass }}
    steps:
      - name: upload info
        env:
          PR_NUMBER: ${{ github.event.number }}
        run: |
          echo $PR_NUMBER > pr_num.txt
          GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha)
          echo ${GITHUB_SHA} > head_sha.txt

      - name: Upload the PR number
        uses: actions/upload-artifact@v2
        with:
          name: pr_num
          path: ./pr_num.txt
          retention-days: 3

      - name: Upload the PR HEAD REF
        uses: actions/upload-artifact@v2
        with:
          name: head_sha
          path: ./head_sha.txt
          retention-days: 3

      - name: Check
        id: check
        if: >
          !contains(github.event.pull_request.title, '(sync #') &&
          !contains(github.event.pull_request.labels.*.name, 'sync') &&
          (!startsWith(github.head_ref, github.base_ref) || !contains(github.head_ref, '-sync-'))
        run: |
          echo "pass=true" >> $GITHUB_OUTPUT

      - name: Check Blacklist
        id: check_blacklist
        env:
          NO_SYNC: ${{ steps.check.outputs.pass }}
          AUTHOR: ${{ github.event.pull_request.user.login }}
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull >/dev/null
          ./scripts/check-blacklist.sh

  be-checker:
    runs-on: ubuntu-latest
    needs: basic-checker
    name: BE FILTER
    if: needs.basic-checker.outputs.PASS == 'true'
    outputs:
      output1: ${{ steps.be-changes-info.outputs.be }}
      output2: ${{ steps.be-changes-info.outputs.thirdparty }}
    steps:
      - uses: dorny/paths-filter@v3
        id: changes
        with:
          filters: |
            be:
            - 'be/**'
            - 'gensrc/**'
            - 'run-be-ut.sh'
            - 'build.sh'
            - 'thirdparty/**'
            - 'docker/dockerfiles/dev-env/dev-env.Dockerfile'
            thirdparty:
            - 'thirdparty/**'
            - 'docker/dockerfiles/dev-env/dev-env.Dockerfile'

      - name: BE CHECK INFO
        id: be-changes-info
        run: |
          echo "be=${{ steps.changes.outputs.be }}" >> $GITHUB_OUTPUT
          echo "thirdparty=${{ steps.changes.outputs.thirdparty }}" >> $GITHUB_OUTPUT

  clang-format:
    runs-on: [self-hosted, light]
    needs: be-checker
    name: Clang-Format
    env:
      PR_NUMBER: ${{ github.event.number }}
    if: ${{ needs.be-checker.outputs.output1 == 'true' }}
    steps:
      - name: clean
        run: |
          rm -rf ${{ github.workspace }}
          mkdir -p ${{ github.workspace }}

      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: BRANCH INFO
        id: branch
        run: |
          echo ${{github.base_ref}}
          echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT

      - name: Checkout PR
        run: |
          BRANCH=${{steps.branch.outputs.branch}}
          git config --global user.name "wanpengfei-git";
          git config --global user.email "wanpengfei91@163.com";
          git checkout $BRANCH;
          git pull;
          BRANCH_NAME="${BRANCH}-${PR_NUMBER}";
          git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME};
          git checkout $BRANCH_NAME;
          git checkout -b merge_pr;
          git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1);

      - name: Run Clang-Format
        run: |
          export PATH=/var/lib/llvm/bin:$PATH
          bash build-support/check-format.sh

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ github.workspace }}/*

  thirdparty-update:
    runs-on: [self-hosted, normal]
    needs: [ be-checker, clang-format ]
    name: Thirdparty Update
    env:
      PR_NUMBER: ${{ github.event.number }}
      BRANCH: ${{ github.base_ref }}
      REPO: ${{ github.repository }}
    continue-on-error: true
    strategy:
      fail-fast: false
      matrix:
        linux: [ centos7, ubuntu ]
    steps:
      - name: clean
        run: |
          rm -rf ${{ github.workspace }}
          mkdir -p ${{ github.workspace }}

      - name: Update Image (${{ matrix.linux }})
        id: update-image
        if: needs.be-checker.outputs.output2  == 'true'
        env:
          linux_distro: ${{ matrix.linux }}
        run: |
          cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          ./bin/run-pr-update-image.sh

      - name: Upload Thirdparty Result
        uses: actions/upload-artifact@v4
        with:
          name: THIRDPARTY-RESULT-${{ matrix.linux }}
          path: image_cache.info
          retention-days: 1
          overwrite: true

      - name: Clean ENV
        if: always()
        run: |
          if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then
              cd ci-tool && source lib/init.sh
              ./bin/elastic-cluster.sh --delete
          fi
          rm -rf ${{ github.workspace }}/*

  thirdparty-info:
    runs-on: [ self-hosted, normal ]
    needs:
      - thirdparty-update
    name: Thirdparty Info
    outputs:
      centos7_image_cache_id: ${{ steps.info.outputs.centos7_image_cache_id }}
      ubuntu_image_cache_id: ${{ steps.info.outputs.ubuntu_image_cache_id }}
    steps:
      - name: Check Result
        run: |
          if [[ "${{ needs.thirdparty-update.result }}" == 'failure' ]]; then
            echo "::error:: Thirdparty Update Error!"
            exit 1
          fi

      - name: Download Thirdparty Artifact
        uses: actions/download-artifact@v4
        with:
          pattern: THIRDPARTY-RESULT-*
          path: outputs

      - name: Read Info
        id: info
        if: needs.thirdparty-update.result == 'success'
        run: |
          image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-centos7/image_cache.info" || echo "")
          echo "centos7_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT
          image_cache_id=$(cat "./outputs/THIRDPARTY-RESULT-ubuntu/image_cache.info" || echo "")
          echo "ubuntu_image_cache_id=${image_cache_id}" >> $GITHUB_OUTPUT

  be-ut:
    runs-on: [self-hosted, normal]
    needs: [ be-checker, thirdparty-info ]
    timeout-minutes: 180
    name: BE UT
    env:
      PR_NUMBER: ${{ github.event.number }}
      BRANCH: ${{ github.base_ref }}
    steps:
      - name: BRANCH INFO
        id: branch
        run: |
          echo ${{github.base_ref}}
          echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT

      - name: UPDATE ECI & RUN UT
        id: run_ut
        shell: bash
        timeout-minutes: 90
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          if [[ "${{ needs.be-checker.outputs.output2 }}" == 'true' ]]; then
              export image_cache_id=${{ needs.thirdparty-info.outputs.centos7_image_cache_id }}
              export image_tag=$BRANCH-$PR_NUMBER
          fi
          ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module be --branch ${{ steps.branch.outputs.branch }} --repository ${{ github.repository }}

      - name: clean ECI
        if: always()
        run: |
          echo ${{ steps.run_ut.outputs.ECI_ID }}
          eci rm ${{ steps.run_ut.outputs.ECI_ID }}

      - name: Upload log
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: BE UT LOG
          path: ${{ steps.run_ut.outputs.BE_LOG }}
          retention-days: 1
          overwrite: true

      - name: Clean ENV
        if: always()
        run: |
          rm -f ${{ steps.run_ut.outputs.RES_FILE }}
          rm -f ${{ steps.run_ut.outputs.RES_LOG }}
          rm -rf ${{ steps.run_ut.outputs.BE_LOG }}
          rm -rf ${{ github.workspace }}/*

  fe-checker:
    runs-on: ubuntu-latest
    needs: basic-checker
    name: FE FILTER
    if: needs.basic-checker.outputs.PASS == 'true'
    outputs:
      output1: ${{ steps.fe-changes-info.outputs.fe }}
    steps:
      - uses: dorny/paths-filter@v3
        id: changes
        with:
          filters: |
            fe:
              - 'fe/**'
              - 'gensrc/**'
              - 'run-fe-ut.sh'
              - 'java-extensions/**'
              - 'build.sh'

      - run: echo ${{ steps.changes.outputs.fe }}

      - name: FE CHECK INFO
        id: fe-changes-info
        run: |
          echo "fe=${{ steps.changes.outputs.fe }}" >> $GITHUB_OUTPUT

  fe-codestyle-check:
    runs-on: ubuntu-latest
    needs: fe-checker
    if: needs.fe-checker.outputs.output1 == 'true'
    name: FE Code Style Check
    env:
      PR_NUMBER: ${{ github.event.number }}
    steps:
      - name: clean
        run: |
          rm -rf ${{ github.workspace }}
          mkdir -p ${{ github.workspace }}

      - uses: dorny/paths-filter@v3
        id: java-file
        name: JAVA FILE
        with:
          filters: |
            java:
              - '**.java'

      - run: echo ${{ steps.java-file.outputs.java }}

      - name: BRANCH INFO
        id: branch
        run: |
          echo ${{github.base_ref}}
          echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Checkout PR
        run: |
          BRANCH=${{steps.branch.outputs.branch}}
          git config --global user.name "wanpengfei-git";
          git config --global user.email "wanpengfei91@163.com";
          git checkout $BRANCH;
          git pull;
          BRANCH_NAME="${BRANCH}-${PR_NUMBER}";
          git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME};
          git checkout $BRANCH_NAME;
          git checkout -b merge_pr;
          git merge --squash --no-edit ${BRANCH} || (echo "::error::Merge conflict, please check." && exit -1);

      - name: Copy checkstyle files
        if: ${{ steps.java-file.outputs.java == 'true' }}
        run: |
          cp fe/checkstyle* .

      - name: Run java checkstyle
        if: ${{ steps.java-file.outputs.java == 'true' }}
        uses: dbelyaev/action-checkstyle@v0.7.5
        with:
          workdir: "./fe"
          checkstyle_config: checkstyle.xml
          reporter: 'github-pr-check'
          github_token: ${{ secrets.GITHUB_TOKEN }}
          fail_on_error: true
          level: error

  sonarcloud-fe-checker:
    runs-on: ubuntu-latest
    needs: fe-checker
    if: needs.fe-checker.outputs.output1 == 'true' && github.repository == 'StarRocks/starrocks'
    name: FE Sonarcloud Check
    env:
      PR_NUMBER: ${{ github.event.number }}
      CODE_PATH: ${{ github.workspace }}
      BRANCH: ${{ github.base_ref }}
      GH_TOKEN: ${{ github.token }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: BRANCH INFO
        id: branch
        run: |
          echo ${{github.base_ref}}
          echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT

      - name: Checkout PR
        run: |
          BRANCH=${{steps.branch.outputs.branch}}
          git config --global user.name "wanpengfei-git";
          git config --global user.email "wanpengfei91@163.com";
          git checkout $BRANCH;
          git pull;
          BRANCH_NAME="${BRANCH}-${PR_NUMBER}";
          git fetch origin pull/${PR_NUMBER}/head:${BRANCH_NAME};
          git checkout $BRANCH_NAME;
          git checkout -b merge_pr;
          git merge --squash --no-edit ${BRANCH} || (echo "Merge conflict, please check." && exit -1);

      - name: Set up JDK 17
        uses: actions/setup-java@v3
        with:
          java-version: 17
          distribution: 'adopt'

      - name: Cache SonarCloud packages
        uses: actions/cache@v3
        with:
          path: ~/.sonar/cache
          key: ${{ runner.os }}-sonar
          restore-keys: ${{ runner.os }}-sonar

      - name: Cache Maven packages
        uses: actions/cache@v3
        with:
          path: ~/.m2
          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
          restore-keys: ${{ runner.os }}-maven

      - name: Setup thrift
        uses: dodopizza/setup-thrift@v1
        with:
          version: 0.13.0

      - name: Analyze FE
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
          SONAR_TOKEN: f0fb4d25c03bae90c2e994c45c29c49dc86fc169 # ${{ secrets.SONAR_TOKEN }}
        run: |
          thrift --version
          whereis thrift
          export STARROCKS_HOME=${{ github.workspace }}
          source env.sh
          mkdir -p thirdparty/installed/bin/
          cd thirdparty/installed/bin/ && ln -s /usr/local/bin/thrift thrift
          cd ${{ github.workspace }}/fe
          mvn -B -DskipTests verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=StarRocks_starrocks -Dsonar.pullrequest.key=${{ github.event.number }} -Dsonar.pullrequest.base=${{ github.base_ref }} -Dsonar.pullrequest.branch=${{ github.head_ref }}

  fe-ut:
    runs-on: [self-hosted, normal]
    needs: fe-codestyle-check
    name: FE UT
    env:
      PR_NUMBER: ${{ github.event.number }}
    steps:
      - name: clean
        run: |
          rm -rf ${{ github.workspace }}
          mkdir -p ${{ github.workspace }}

      - name: BRANCH INFO
        id: branch
        run: |
          echo "branch=${{github.base_ref}}" >> $GITHUB_OUTPUT
          repo="${{ github.repository }}"
          bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'` 
          echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT

      - name: UPDATE ECI & RUN UT
        id: run_ut
        shell: bash
        timeout-minutes: 60
        run: |
          rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
          ./bin/elastic-ut.sh --pr ${PR_NUMBER} --module fe --branch ${{steps.branch.outputs.branch}} --build Release --repository ${{ github.repository }}

      - name: Clean ECI
        if: always()
        run: |
          echo ${{ steps.run_ut.outputs.ECI_ID }}
          echo ">>> Dmesg info:"
          eci exec ${{ steps.run_ut.outputs.ECI_ID }} bash -c "dmesg -T"
          eci rm ${{ steps.run_ut.outputs.ECI_ID }}

      - name: Upload log
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: FE UT LOG
          path: ${{ steps.run_ut.outputs.RES_LOG }}
          retention-days: 1
          overwrite: true

      - name: Clean ENV
        if: always()
        run: |
          rm -rf ${{ steps.run_ut.outputs.FE_REPORT_DIR }}
          rm -f ${{ steps.run_ut.outputs.RES_FILE }}
          rm -f ${{ steps.run_ut.outputs.RES_LOG }}
          rm -rf ${{ steps.run_ut.outputs.COV_DIR }}
          rm -rf ${{ github.workspace }}/*

  Teardown:
    runs-on: [self-hosted, normal]
    name: Teardown
    needs: [ fe-ut, be-ut ]
    if: always()
    env:
      PR_NUMBER: ${{ github.event.number }}
      BRANCH: ${{ github.base_ref }}
    steps:
      - name: upload info
        if: always()
        run: |
          echo $PR_NUMBER > pr_num.txt
          GITHUB_SHA=$(cat $GITHUB_EVENT_PATH | jq -r .pull_request.head.sha)
          echo ${GITHUB_SHA} > head_sha.txt

      - name: Upload the PR number
        uses: actions/upload-artifact@v4
        with:
          name: pr_num
          path: ./pr_num.txt
          retention-days: 3
          overwrite: true

      - name: Upload the PR HEAD REF
        uses: actions/upload-artifact@v4
        with:
          name: head_sha
          path: ./head_sha.txt
          retention-days: 3

      - name: Clean
        run: |
          rm -rf ${{ github.workspace }}/*