DRAFT: ci(runner): Add benchmark example and print benchmark results. #118
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build ESP-BSP apps | |
| # This job builds all examples and test_applications in this repo | |
| # Applications are selected for build based on changes files and dependencies defined in .build-test-rules.yml | |
| on: | |
| pull_request: | |
| types: [opened, reopened, synchronize, labeled] | |
| workflow_dispatch: | |
| inputs: | |
| WFType: | |
| description: 'Workflow type' | |
| required: true | |
| default: 'Build + Tests' | |
| type: choice | |
| options: | |
| - Build + Tests | |
| - Build + Tests + Benchmark | |
| jobs: | |
| build: | |
| strategy: | |
| matrix: | |
| include: | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 1 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 2 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 3 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 4 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 5 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 6 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 7 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 8 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 9 | |
| - idf_ver: "latest" | |
| parallel_count: 10 | |
| parallel_index: 10 | |
| #- idf_ver: "release-v5.1" | |
| # parallel_count: 2 | |
| # parallel_index: 1 | |
| #- idf_ver: "release-v5.1" | |
| # parallel_count: 2 | |
| # parallel_index: 2 | |
| #- idf_ver: "release-v5.2" | |
| # parallel_count: 2 | |
| # parallel_index: 1 | |
| #- idf_ver: "release-v5.2" | |
| # parallel_count: 2 | |
| # parallel_index: 2 | |
| #- idf_ver: "release-v5.3" | |
| # parallel_count: 2 | |
| # parallel_index: 1 | |
| #- idf_ver: "release-v5.3" | |
| # parallel_count: 2 | |
| # parallel_index: 2 | |
| #- idf_ver: "release-v5.4" | |
| # parallel_count: 2 | |
| # parallel_index: 1 | |
| #- idf_ver: "release-v5.4" | |
| # parallel_count: 2 | |
| # parallel_index: 2 | |
| runs-on: ubuntu-latest | |
| container: espressif/idf:${{ matrix.idf_ver }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get changed files | |
| id: changed-files | |
| uses: tj-actions/changed-files@v45 | |
| with: | |
| separator: ';' # idf-build-apps expects files seprated with semicolon | |
| - name: Build ESP-BSP applications | |
| shell: bash | |
| env: | |
| IDF_EXTRA_ACTIONS_PATH: "${{ github.workspace }}/examples" | |
| ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_modified_files }} | |
| run: | | |
| . ${IDF_PATH}/export.sh | |
| pip install idf-component-manager==1.* ruamel.yaml idf-build-apps==2.4.3 --upgrade | |
| echo "Files changed: ${ALL_CHANGED_FILES}" | |
| idf-build-apps find --modified-files "${ALL_CHANGED_FILES}" | |
| idf-build-apps build --parallel-count ${{ matrix.parallel_count }} --parallel-index ${{ matrix.parallel_index }} --modified-files "${ALL_CHANGED_FILES}" --collect-app-info build_info_${{ matrix.idf_ver }}_${{ matrix.parallel_index }}.json | |
| - uses: actions/upload-artifact@v4 | |
| if: github.repository_owner == 'espressif' && needs.prepare.outputs.build_only == '0' | |
| with: | |
| name: app_binaries_${{ matrix.idf_ver }}_${{ matrix.parallel_index }} | |
| path: | | |
| examples/*/build_*/bootloader/bootloader.bin | |
| examples/*/build_*/partition_table/partition-table.bin | |
| examples/*/build_*/*.bin | |
| examples/*/build_*/flasher_args.json | |
| examples/*/build_*/config/sdkconfig.json | |
| build_info*.json | |
| run-target: | |
| name: Run apps | |
| if: github.repository_owner == 'espressif' && !contains(github.event.pull_request.labels.*.name, 'Build only') | |
| needs: build | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 5 | |
| matrix: | |
| idf_ver: | |
| - "latest" | |
| runner: | |
| # - runs-on: "esp-box-3" | |
| # marker: "esp_box_3" | |
| # target: "esp32s3" | |
| # - runs-on: "esp32_c3_lcdkit" | |
| # marker: "esp32_c3_lcdkit" | |
| # target: "esp32c3" | |
| # - runs-on: "esp32_p4_box" | |
| # marker: "esp32_p4_box" | |
| # target: "esp32p4" | |
| - runs-on: "esp32_p4_function_ev_board" | |
| marker: "esp32_p4_function_ev_board" | |
| target: "esp32p4" | |
| # - runs-on: "esp32_s3_eye" | |
| # marker: "esp32_s3_eye" | |
| # target: "esp32s3" | |
| - runs-on: "esp32_s3_lcd_ev_board" | |
| marker: "esp32_s3_lcd_ev_board" | |
| target: "esp32s3" | |
| - runs-on: "esp32_s3_lcd_ev_board" | |
| marker: "esp32_s3_lcd_ev_board_2" | |
| target: "esp32s3" | |
| # - runs-on: "esp32_s3_usb_otg" | |
| # marker: "esp32_s3_usb_otg" | |
| # target: "esp32s3" | |
| # - runs-on: "esp_wrover_kit" | |
| # marker: "esp_wrover_kit" | |
| # target: "esp32" | |
| # - runs-on: "m5dial" | |
| # marker: "m5dial" | |
| # target: "esp32s3" | |
| # - runs-on: "m5stack_core" | |
| # marker: "m5stack_core" | |
| # target: "esp32" | |
| # - runs-on: "m5stack_core_2" | |
| # marker: "m5stack_core_2" | |
| # target: "esp32" | |
| # - runs-on: "m5stack_core_s3" | |
| # marker: "m5stack_core_s3" | |
| # target: "esp32s3" | |
| # - runs-on: "m5stack_core_s3" | |
| # marker: "m5stack_core_s3_se" | |
| # target: "esp32s3" | |
| # - runs-on: "esp32_azure_iot_kit" | |
| # marker: "esp32_azure_iot_kit" | |
| # target: "esp32" | |
| env: | |
| TEST_RESULT_NAME: test_results_${{ matrix.runner.target }}_${{ matrix.runner.marker }}_${{ matrix.idf_ver }} | |
| TEST_RESULT_FILE: test_results_${{ matrix.runner.target }}_${{ matrix.runner.marker }}_${{ matrix.idf_ver }}.xml | |
| runs-on: [self-hosted, Linux, bspwall] | |
| container: | |
| image: python:3.11-bookworm | |
| options: --privileged -v /dev/boards:/dev/boards/ # Privileged mode has access to serial ports | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| pattern: app_binaries_${{ matrix.idf_ver }}_* | |
| merge-multiple: true | |
| - name: Install Python packages | |
| env: | |
| PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi/" | |
| run: | | |
| echo "PYTEST_BENCHMARK_IGNORE=--ignore='examples/display_lvgl_benchmark'" >> $GITHUB_ENV | |
| pip install --prefer-binary cryptography pytest-embedded pytest-embedded-serial-esp pytest-embedded-idf pytest-custom_exit_code | |
| - name: Set ignores | |
| if: contains(github.event.pull_request.labels.*.name, 'Run benchmark') || contains(inputs.WFType, 'Build + Tests + Benchmark') | |
| id: set_ignores | |
| run: | | |
| echo "PYTEST_BENCHMARK_IGNORE=" >> $GITHUB_ENV | |
| - name: Run apps | |
| run: | | |
| pytest --suppress-no-test-exit-code --ignore-glob '*/managed_components/*' --ignore=.github ${{ env.PYTEST_BENCHMARK_IGNORE }} --junit-xml=${{ env.TEST_RESULT_FILE }} --target=${{ matrix.runner.target }} -m ${{ matrix.runner.marker }} --build-dir=build_${{ matrix.runner.runs-on }} | |
| - name: Check Git status | |
| run: | | |
| git config --global --add safe.directory /__w/esp-bsp/esp-bsp | |
| git status | |
| - name: Commit changes | |
| uses: EndBug/add-and-commit@v9 | |
| with: | |
| author_name: github-actions[bot] | |
| author_email: github-actions[bot]@users.noreply.github.com | |
| message: '🔄 Benchmark results' | |
| add: bsp/* | |
| # - name: Commit test results | |
| # run: | | |
| # git config --global user.name "github-actions[bot]" | |
| # git config --global user.email "github-actions[bot]@users.noreply.github.com" | |
| # git add bsp/* | |
| # git commit -m "🔄 Benchmark results" | |
| # git push origin HEAD:${{ github.head_ref }} | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: ${{ env.TEST_RESULT_NAME }} | |
| path: | | |
| ${{ env.TEST_RESULT_FILE }} | |
| benchmark_*.md | |
| benchmark_*.json | |
| benchmark.json | |
| publish-results: | |
| name: Publish Test results | |
| needs: | |
| - run-target | |
| if: github.repository_owner == 'espressif' && always() && github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'Build only') | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Download Test results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: test_results_* | |
| path: test_results | |
| - name: Publish Test Results | |
| uses: EnricoMi/publish-unit-test-result-action@v2 | |
| with: | |
| files: test_results/**/*.xml | |
| - name: Find test result files | |
| id: find_files | |
| run: | | |
| OUTPUT_FILE="combined_benchmarks.md" | |
| echo "" > $OUTPUT_FILE | |
| python <<EOF | |
| import glob | |
| files = sorted(glob.glob("test_results/**/benchmark_*.md")) | |
| print(files) | |
| output_file = "combined_benchmarks.md" | |
| with open(output_file, "w", encoding="utf-8") as outfile: | |
| for file in files: | |
| with open(file, "r", encoding="utf-8") as infile: | |
| outfile.write(infile.read() + "\n\n") | |
| print(f"Merged {len(files)} files into {output_file}") | |
| EOF | |
| echo "output_file=$OUTPUT_FILE" >> "$GITHUB_ENV" | |
| - name: Comment PR | |
| uses: thollander/actions-comment-pull-request@v3 | |
| with: | |
| comment-tag: benchmark_results | |
| file-path: ${{ env.output_file }} |