WeWork Robot #373
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: WeWork Robot | |
| # Controls when the workflow will run | |
| on: | |
| issues: | |
| types: [closed, assigned, opened] | |
| schedule: | |
| - cron: '30 8 * * *' | |
| issue_comment: | |
| types: [created] | |
| workflow_dispatch: | |
| # A workflow run is made up of one or more jobs that can run sequentially or in parallel. | |
| jobs: | |
| issue-notify: | |
| if : github.event_name == 'issues' && github.event.action == 'assigned' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name : ISSUE Triggered | |
| uses: chf007/action-wechat-work@master | |
| env: | |
| WECHAT_WORK_BOT_WEBHOOK: ${{secrets.WECHAT_WORK_BOT_WEBHOOK}} | |
| with: | |
| msgtype: markdown | |
| mentioned_list: '["@all"]' | |
| content: "New Issue Opened or Assigned: [${{ github.event.issue.title }}](${{ github.event.issue.html_url }})\nAssignees:<font color=\"warning\">${{ join(github.event.issue.assignees.*.login, ', ') }}</font>\nOpen Issue Number: <font color=\"warning\">${{ github.event.repository.open_issues }}</font>" | |
| add-projects-milestone: | |
| if: github.event_name == 'issues' && github.event.action == 'opened' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Issue Add Projects | |
| if: contains(github.event.issue.labels.*.name, 'kind/bug') | |
| uses: actions/[email protected] | |
| with: | |
| project-url: https://github.com/orgs/matrixorigin/projects/13 | |
| github-token: ${{ secrets.TOKEN_ACTION }} | |
| labeled: kind/bug | |
| label-operator: OR | |
| - name: Issue Add Milestone | |
| if: contains(github.event.issue.labels.*.name, 'kind/bug') | |
| uses: guguducken/milestone-add-action@check_mill_exist | |
| with: | |
| action-token: ${{ secrets.TOKEN_ACTION }} | |
| milestone: "1.0.0" | |
| check-bvt-issue: | |
| if: github.event_name == 'issues' && github.event.action == 'closed' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: CheckOut | |
| uses: actions/checkout@v3 | |
| - name: Check the code to see if the issue number is removed | |
| id: get_check_val | |
| env: | |
| ISSUE_NUM: ${{ github.event.issue.number }} | |
| run: | | |
| res=`grep -r "issue#$ISSUE_NUM" ${{ github.workspace }}/test || true` | |
| if [ -n "$res" ]; then | |
| echo "::set-output name=check_val::True" | |
| else | |
| echo "::set-output name=check_val::False" | |
| fi | |
| - name: Reopen Issue | |
| if: ${{ steps.get_check_val.outputs.check_val == 'True' }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: 'open-issue' | |
| token: ${{ secrets.TOKEN_ACTION }} | |
| issue-number: ${{ github.event.issue.number }} | |
| - name: create-comment | |
| if: ${{ steps.get_check_val.outputs.check_val == 'True' }} | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: 'create-comment' | |
| token: ${{ secrets.TOKEN_ACTION }} | |
| issue-number: ${{ github.event.issue.number }} | |
| body: | | |
| Hello @${{ github.event.issue.user.login }}. The bug issue in the BVT test code has not been removed,issues automatically open. | |
| issue-reopen: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'issues' && github.event.action == 'closed' | |
| steps: | |
| - name: Reopen Issue | |
| if: github.event.issue.user.login != github.event.sender.login && github.event.action == 'closed' && github.event.sender.login != 'sukki37' && github.event.sender.login != 'aressu1985' && github.event.sender.login != 'fengttt' | |
| uses: actions-cool/issues-helper@v3 | |
| with: | |
| actions: 'open-issue' | |
| token: ${{ secrets.TOKEN_ACTION }} | |
| issue-number: ${{ github.event.issue.number }} | |
| issue_close_check: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'issues' && github.event.action == 'closed' | |
| steps: | |
| - uses: guguducken/pull-issue-reviewer@issue_close_check_with_name | |
| with: | |
| token_action: ${{ secrets.TOKEN_ACTION }} | |
| uri_notice: ${{ secrets.DOCU_GROUP_HOOK }} | |
| name_label: "doc influence" | |
| mentions: "Prinz,DengNan" | |
| auto-to-project: | |
| runs-on: ubuntu-latest | |
| if: github.event_name == 'issues' && github.event.action == 'opened' | |
| steps: | |
| - name: Add Issue To GitHub Project | |
| uses: actions/[email protected] | |
| with: | |
| project-url: https://github.com/orgs/matrixorigin/projects/13/views/1 | |
| github-token: ${{ secrets.TOKEN_ACTION }} | |
| labeled: kind/bug,kind/docs,kind/enhancement,kind/feature,kind/questions,kind/performance,kind/refactoring,kind/subtask | |
| label-operator: OR | |
| nightly-check: | |
| if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} | |
| runs-on: aws-arm64-runner | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: '3' | |
| path: ./matrixone | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: '1.20' | |
| cache: true | |
| - name: Set Variables | |
| run: | | |
| echo "ut_report='UT-Report.out'" >> $GITHUB_ENV | |
| echo "raw_ut_coverage='ut_coverage.raw'" >> $GITHUB_ENV | |
| echo "raw_bvt_coverage='bvt_coverage.raw'" >> $GITHUB_ENV | |
| echo "ut_coverage='ut_coverage.out'" >> $GITHUB_ENV | |
| echo "bvt_coverage='bvt_coverage.out'" >> $GITHUB_ENV | |
| echo "ut_html_coverage='ut_coverage.html'" >> $GITHUB_ENV | |
| echo "bvt_html_coverage='bvt_coverage.html'" >> $GITHUB_ENV | |
| echo "ut_excluded_pkg='pkg/pb\|pkg/sql/parsers/goyacc\|yaccpar'" >> $GITHUB_ENV | |
| echo "bvt_excluded_pkg='pkg/pb\|yaccpar'" >> $GITHUB_ENV | |
| - name: Clone test-tool repository | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: matrixorigin/mo-tester | |
| path: ./mo-tester | |
| ref: main | |
| - name: Set up JDK 8 for x64 | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: '8' | |
| distribution: 'adopt' | |
| - name: Start Unit Test | |
| timeout-minutes: 120 | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| test_scope=$(go list ./... | grep -v 'driver\|engine/aoe\|engine/memEngine\|pkg/catalog') | |
| make clean && make config && make cgo | |
| echo "ut start" | |
| CGO_CFLAGS="-I${GITHUB_WORKSPACE}/matrixone/cgo" CGO_LDFLAGS="-L${GITHUB_WORKSPACE}/matrixone/cgo -lmo" go test -v -tags matrixone_test -p 6 -covermode=count -coverprofile=${{ env.raw_ut_coverage }} -coverpkg=./pkg/... ${test_scope} | tee ${{ env.ut_report }} | |
| echo "ut finished" | |
| - name: Start BVT Test Service | |
| if: ${{ always() && !cancelled() }} | |
| timeout-minutes: 10 | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| sed -i.bak 's/level = "info"/level = "error"/g' ./etc/launch/log.toml | |
| if [ -d "./store" ]; then rm -rf ./store; fi | |
| GOLDFLAGS=-ldflags="-X 'main.GoVersion=$(go version)' -X 'main.BranchName=$(git rev-parse --abbrev-ref HEAD)' -X 'main.CommitID=$(git rev-parse HEAD)' -X 'main.BuildTime=$(date)' -X 'main.Version=$(git describe --always --tags $(git rev-list --tags --max-count=1))'" | |
| export GOCOVERDIR=$GITHUB_WORKSPACE/matrixone | |
| go build -cover "$GOLDFLAGS" -o mo-service ./cmd/mo-service/ | |
| ./optools/run_bvt.sh $GITHUB_WORKSPACE/matrixone launch | |
| - name: Start BVT Test | |
| if: ${{ always() && !cancelled() }} | |
| id: bvt_on_pr_version | |
| run: | | |
| export LC_ALL="C.UTF-8" | |
| locale | |
| cd $GITHUB_WORKSPACE/mo-tester | |
| ./run.sh -n -g -p $GITHUB_WORKSPACE/matrixone/test/distributed/cases -e optimistic 2>&1 | |
| - name: Generate UT and BVT Coverage File | |
| if: ${{ always() && !cancelled() }} | |
| timeout-minutes: 10 | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| cat ${{ env.raw_ut_coverage }} | grep -v ${{ env.ut_excluded_pkg }} > ${{ env.ut_coverage }} | |
| echo "ut_coverage finished" | |
| go tool cover -o ${{ env.ut_html_coverage }} -html=${{ env.ut_coverage }} | |
| echo "ut_html_coverage finished" | |
| rm -rf ${{ env.raw_ut_coverage }} | |
| ps -aux | grep 'mo-service' | grep -v 'grep' | awk '{print $2}' | xargs kill -s SIGINT | |
| i=1 && while [ -n "$(ps -aux|grep 'mo-service' | grep -v 'grep' | awk '{print $2}')" ] && [ $i -le 300 ]; do echo "mo-service kill not finished......$i/300"; i=$(($i+1)); sleep 1s; done | |
| if [ $i -gt 300 ]; then echo 'stop mo-service failed...'; exit 1; else echo 'stop mo-service successed...'; fi | |
| go tool covdata textfmt -i="$GITHUB_WORKSPACE/matrixone" -o ${{ env.raw_bvt_coverage }} | |
| cat ${{ env.raw_bvt_coverage }} | grep -v ${{ env.bvt_excluded_pkg }} > ${{ env.bvt_coverage }} | |
| echo "bvt_coverage finished" | |
| go tool cover -o ${{ env.bvt_html_coverage }} -html=${{ env.bvt_coverage }} | |
| echo "bvt_html_coverage finished" | |
| - name: Coverage Calculate | |
| if: ${{ always() && !cancelled() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| echo "UT_CODE_COVERAGE: $(awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}' ${{ env.ut_coverage }})" > coverage_report.txt | |
| echo "BVT_CODE_COVERAGE: $(awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}' ${{ env.bvt_coverage }})" >> coverage_report.txt | |
| echo "UT_CODE_COVERAGE=$(awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}' ${{ env.ut_coverage }})" >> $GITHUB_ENV | |
| echo "BVT_CODE_COVERAGE=$(awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}' ${{ env.bvt_coverage }})" >> $GITHUB_ENV | |
| coverage="$(cat ${{ env.ut_coverage }} ${{ env.bvt_coverage }} | awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}')" | |
| echo "Overall Coverage: ${coverage}" >> coverage_report.txt | |
| echo "GENERAL_CODE_COVERAGE=${coverage}" >> $GITHUB_ENV | |
| pkgs=$(go list ./pkg/... | awk -v FS='/' -v OFS='/' '{if ($5=="vm" || $5=="sql") print $4,$5,$6"/"; else print $4,$5"/"}' | sort -u | grep -v ${{ env.ut_excluded_pkg }} | grep -v '//') | |
| echo -e "\n------------ut detail coverage----------" >> coverage_report.txt | |
| echo "" > ut_tmp.txt | |
| for pkg in ${pkgs[@]} | |
| do | |
| if [ "$(cat ${{ env.ut_coverage }} |grep $pkg | grep -v "grep" | wc -l)" -gt 0 ]; | |
| then | |
| c="$(cat ${{ env.ut_coverage }} |grep $pkg|awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}')"; | |
| echo -e "${pkg}:\t${c}" >> ut_tmp.txt; | |
| else | |
| echo -e "${pkg}:\t0" >> ut_tmp.txt; | |
| fi | |
| done | |
| cat ut_tmp.txt | sort -k 1 -k 2n >> coverage_report.txt | |
| pkgs=$(go list ./pkg/... | awk -v FS='/' -v OFS='/' '{if ($5=="vm" || $5=="sql") print $4,$5,$6"/"; else print $4,$5"/"}' | sort -u | grep -v ${{ env.bvt_excluded_pkg }} | grep -v '//') | |
| echo -e "\n------------bvt detail coverage----------" >> coverage_report.txt | |
| echo "" > bvt_tmp.txt | |
| for pkg in ${pkgs[@]} | |
| do | |
| if [ "$(cat ${{ env.bvt_coverage }} |grep $pkg | grep -v "grep" | wc -l)" -gt 0 ]; | |
| then | |
| c="$(cat ${{ env.bvt_coverage }} |grep $pkg|awk 'BEGIN {covered_line=0;all_line=0} {if ($3>=1 && !($1 in covered)) {covered[$1] = $2; covered_line+=$2;} if (!($1 in all)) { all[$1] = $2; all_line+=$2;}} END {print covered_line/all_line}')"; | |
| echo -e "${pkg}:\t${c}" >> bvt_tmp.txt; | |
| else | |
| echo -e "${pkg}:\t0" >> bvt_tmp.txt; | |
| fi | |
| done | |
| cat bvt_tmp.txt | sort -k 1 -k 2n >> coverage_report.txt | |
| # zip -r code_quality_reports.zip ${{ env.bvt_html_coverage }} ${{ env.ut_html_coverage }} coverage_report.txt lint.out UT-Report.out | |
| zip -r code_quality_reports.zip ${{ env.bvt_html_coverage }} ${{ env.ut_html_coverage }} coverage_report.txt UT-Report.out $GITHUB_WORKSPACE/matrixone/mo-service.log | |
| - name: Generate Notice Content | |
| if: ${{ always() && !cancelled() }} | |
| run: | | |
| cd $GITHUB_WORKSPACE/matrixone | |
| cat coverage_report.txt | |
| echo "Today=$(date -d now +%Y-%m-%d)" >> $GITHUB_ENV | |
| echo "Url=${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV | |
| echo 'Detail<<EOF' >> $GITHUB_ENV | |
| cat coverage_report.txt >> $GITHUB_ENV | |
| echo 'EOF' >> $GITHUB_ENV | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ always() && !cancelled() }} | |
| continue-on-error: true | |
| with: | |
| name: daily-code-quality-reports | |
| path: | | |
| ${{ github.workspace }}/matrixone/code_quality_reports.zip | |
| retention-days: 7 | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| with: | |
| name: daily-test-reports | |
| path: | | |
| ${{ github.workspace }}/matrixone | |
| retention-days: 7 | |
| - name: WeChat Work notification | |
| if: ${{ always() && !cancelled() }} | |
| id: notification | |
| uses: chf007/action-wechat-work@master | |
| env: | |
| WECHAT_WORK_BOT_WEBHOOK: ${{secrets.WECHAT_WORK_BOT_WEBHOOK}} | |
| with: | |
| msgtype: markdown | |
| mentioned_list: '["@all"]' | |
| content: "<font color=\"warning\">Nightly Code Quality - ${{ env.Today }} </font>\nUT Coverage Summary:<font color=\"warning\">${{ env.UT_CODE_COVERAGE }}</font>\nBVT Coverage Summary:<font color=\"warning\">${{ env.BVT_CODE_COVERAGE }}</font>\nGeneral Coverage Summary:<font color=\"warning\">${{ env.GENERAL_CODE_COVERAGE }}</font>\nFor details, please visit [Action Run](${{ env.Url }})\n" | |
| - name: WeChat Work notification | |
| if: ${{ always() && !cancelled() }} | |
| id: detail-notification | |
| uses: chf007/action-wechat-work@master | |
| env: | |
| WECHAT_WORK_BOT_WEBHOOK: ${{secrets.WECHAT_WORK_BOT_WEBHOOK}} | |
| with: | |
| msgtype: text | |
| mentioned_list: '["@all"]' | |
| content: ${{ env.Detail }} | |