Merge pull request #64950 from ClickHouse/ci_more_fixes_for_sync_prs #197
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # yamllint disable rule:comments-indentation | |
| name: MasterCI | |
| env: | |
| # Force the stdout and stderr streams to be unbuffered | |
| PYTHONUNBUFFERED: 1 | |
| on: # yamllint disable-line rule:truthy | |
| push: | |
| branches: | |
| - 'master' | |
| jobs: | |
| RunConfig: | |
| runs-on: [self-hosted, style-checker-aarch64] | |
| outputs: | |
| data: ${{ steps.runconfig.outputs.CI_DATA }} | |
| steps: | |
| - name: DebugInfo | |
| uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6 | |
| - name: Check out repository code | |
| uses: ClickHouse/checkout@v1 | |
| with: | |
| clear-repository: true # to ensure correct digests | |
| fetch-depth: 0 # to get version | |
| filter: tree:0 | |
| - name: Merge sync PR | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 sync_pr.py --merge || : | |
| # Runs in MQ: | |
| # - name: Python unit tests | |
| # run: | | |
| # cd "$GITHUB_WORKSPACE/tests/ci" | |
| # echo "Testing the main ci directory" | |
| # python3 -m unittest discover -s . -p 'test_*.py' | |
| # for dir in *_lambda/; do | |
| # echo "Testing $dir" | |
| # python3 -m unittest discover -s "$dir" -p 'test_*.py' | |
| # done | |
| - name: PrepareRunConfig | |
| id: runconfig | |
| run: | | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json | |
| echo "::group::CI configuration" | |
| python3 -m json.tool ${{ runner.temp }}/ci_run_data.json | |
| echo "::endgroup::" | |
| { | |
| echo 'CI_DATA<<EOF' | |
| cat ${{ runner.temp }}/ci_run_data.json | |
| echo 'EOF' | |
| } >> "$GITHUB_OUTPUT" | |
| - name: Re-create GH statuses for skipped jobs if any | |
| run: | | |
| python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses | |
| # Runs in MQ: | |
| # BuildDockers: | |
| # needs: [RunConfig] | |
| # if: ${{ !failure() && !cancelled() }} | |
| # uses: ./.github/workflows/reusable_docker.yml | |
| # with: | |
| # data: ${{ needs.RunConfig.outputs.data }} | |
| # StyleCheck: | |
| # needs: [RunConfig, BuildDockers] | |
| # if: ${{ !failure() && !cancelled() }} | |
| # uses: ./.github/workflows/reusable_test.yml | |
| # with: | |
| # test_name: Style check | |
| # runner_type: style-checker | |
| # data: ${{ needs.RunConfig.outputs.data }} | |
| # run_command: | | |
| # python3 style_check.py --no-push | |
| ################################# Main stages ################################# | |
| # for main CI chain | |
| # | |
| Builds_1: | |
| needs: [RunConfig] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }} | |
| # using callable wf (reusable_stage.yml) allows grouping all nested jobs under a tab | |
| uses: ./.github/workflows/reusable_build_stage.yml | |
| with: | |
| stage: Builds_1 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| Tests_1: | |
| needs: [RunConfig, Builds_1] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }} | |
| uses: ./.github/workflows/reusable_test_stage.yml | |
| with: | |
| stage: Tests_1 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| Builds_2: | |
| needs: [RunConfig, Builds_1] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }} | |
| uses: ./.github/workflows/reusable_build_stage.yml | |
| with: | |
| stage: Builds_2 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| Tests_2: | |
| needs: [RunConfig, Builds_2] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }} | |
| uses: ./.github/workflows/reusable_test_stage.yml | |
| with: | |
| stage: Tests_2 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| # stage for jobs that do not prohibit merge | |
| Tests_3: | |
| # Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there. | |
| needs: [RunConfig, Builds_1, Builds_2] | |
| if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }} | |
| uses: ./.github/workflows/reusable_test_stage.yml | |
| with: | |
| stage: Tests_3 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| ################################# Reports ################################# | |
| # Reports should be run even if Builds_1/2 failed - put them separately in wf (not in Tests_1/2) | |
| Builds_1_Report: | |
| # run report check for failed builds to indicate the CI error | |
| if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }} | |
| needs: [RunConfig, Builds_1] | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: ClickHouse build check | |
| runner_type: style-checker-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| Builds_2_Report: | |
| # run report check for failed builds to indicate the CI error | |
| if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }} | |
| needs: [RunConfig, Builds_2] | |
| uses: ./.github/workflows/reusable_test.yml | |
| with: | |
| test_name: ClickHouse special build check | |
| runner_type: style-checker-aarch64 | |
| data: ${{ needs.RunConfig.outputs.data }} | |
| MarkReleaseReady: | |
| if: ${{ !failure() && !cancelled() }} | |
| needs: [RunConfig, Builds_1, Builds_2] | |
| runs-on: [self-hosted, style-checker-aarch64] | |
| steps: | |
| - name: Debug | |
| run: | | |
| echo need with different filters | |
| cat << 'EOF' | |
| ${{ toJSON(needs) }} | |
| ${{ toJSON(needs.*.result) }} | |
| no failures ${{ !contains(needs.*.result, 'failure') }} | |
| no skips ${{ !contains(needs.*.result, 'skipped') }} | |
| no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| EOF | |
| - name: Not ready | |
| # fail the job to be able to restart it | |
| if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }} | |
| run: exit 1 | |
| - name: Check out repository code | |
| if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| uses: ClickHouse/checkout@v1 | |
| - name: Mark Commit Release Ready | |
| if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }} | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 mark_release_ready.py | |
| FinishCheck: | |
| if: ${{ !cancelled() }} | |
| needs: [RunConfig, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3] | |
| runs-on: [self-hosted, style-checker-aarch64] | |
| steps: | |
| - name: Check out repository code | |
| uses: ClickHouse/checkout@v1 | |
| - name: Finish label | |
| run: | | |
| cd "$GITHUB_WORKSPACE/tests/ci" | |
| python3 finish_check.py |