Bumps version to 0.11.0. #168
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Performance Regression | |
| on: | |
| pull_request: | |
| paths: | |
| # Workflow will only run if something in this path is changed | |
| - 'amazon/**' | |
| env: | |
| report_statistics: 'file_size,time_mean,time_error,ops/s_mean,ops/s_error,memory_usage_peak' | |
| compare_statistics: 'file_size,time_mean' | |
| data_size: '100' | |
| spec_defaults: '{warmups:100,iterations:100}' | |
| specs: '{command:read,format:ion_text} {command:write,format:ion_text} {command:read,format:ion_binary} {command:write,format:ion_binary}' | |
| test_data_id: 'generated-test-data' | |
| run_cli: 'python amazon/ionbenchmark/ion_benchmark_cli.py' | |
| jobs: | |
| generate-test-data: | |
| name: Generate Data | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Generates data used for benchmarking | |
| - name: Checkout ion-data-generator | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: amazon-ion/ion-data-generator | |
| ref: main | |
| - name: Build ion-data-generator | |
| run: mvn clean install | |
| - name: Generate test Ion Data | |
| env: | |
| jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar | |
| schema_dir: tst/com/amazon/ion/workflow | |
| run: | | |
| mkdir -p testData | |
| for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03 | |
| do | |
| java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n | |
| done | |
| - name: Upload test Ion Data to artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{env.test_data_id}} | |
| path: testData | |
| prepopulate-pip-cache: | |
| # Since all the "Check" jobs can run in parallel, caching _could_ have basically no effect. In order to speed things | |
| # up, this step can run in parallel with the "Generate Data" job, pre-caching all the dependencies. | |
| name: Setup PIP Cache | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| cache-dependency-path: '**/requirements.txt' | |
| - run: | | |
| pip install -r requirements.txt | |
| # TODO: See if there's a way to cache the ion-c build output if it hasn't changed | |
| detect-regression: | |
| name: Check | |
| runs-on: ubuntu-latest | |
| needs: [generate-test-data, prepopulate-pip-cache] | |
| strategy: | |
| matrix: | |
| python-version: ['3.9', '3.11', 'pypy-3.8', 'pypy-3.10'] | |
| test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03'] | |
| fail-fast: false | |
| steps: | |
| - name: Checkout the base of the PR | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.base_ref }} | |
| submodules: recursive | |
| path: baseline | |
| - name: Checkout the head of the PR | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.head_ref }} | |
| submodules: recursive | |
| path: new | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| cache: 'pip' | |
| cache-dependency-path: '**/requirements.txt' | |
| - name: Download Test Data | |
| id: 'download' | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{env.test_data_id}} | |
| # Generates performance results for the previous commit | |
| - name: Create a virtual environment | |
| working-directory: ./baseline | |
| run: | | |
| pip install -r requirements.txt | |
| pip install . | |
| - name: Run baseline performance benchmark | |
| id: 'baseline' | |
| working-directory: ./baseline | |
| run: | | |
| ${{env.run_cli}} spec '${{env.specs}}' -d '${{env.spec_defaults}}' \ | |
| -O '{input_file:"${{steps.download.outputs.download-path}}/${{ matrix.test-data }}.10n"}' \ | |
| -o "$PWD/report.ion" -r '${{env.report_statistics}}' | |
| echo "::group::Ion Report" | |
| echo "$(<report.ion)" | |
| echo "::endgroup::" | |
| echo "report=$PWD/report.ion" >> "$GITHUB_OUTPUT" | |
| # Generates performance results for the current commit | |
| - name: Create a virtual environment and setup the package | |
| working-directory: ./new | |
| run: | | |
| pip install -r requirements.txt | |
| pip install . | |
| - name: Run new performance benchmark | |
| id: 'new' | |
| working-directory: ./new | |
| run: | | |
| ${{env.run_cli}} spec '${{env.specs}}' -d '${{env.spec_defaults}}' \ | |
| -O '{input_file:"${{steps.download.outputs.download-path}}/${{ matrix.test-data }}.10n"}' \ | |
| -o "$PWD/report.ion" -r '${{env.report_statistics}}' | |
| echo "::group::Ion Report" | |
| echo "$(<report.ion)" | |
| echo "::endgroup::" | |
| echo "report=$PWD/report.ion" >> "$GITHUB_OUTPUT" | |
| # Compare results and identify regression | |
| - name: Detect performance regression | |
| working-directory: ./new | |
| run: ${{env.run_cli}} compare --fail ${{steps.baseline.outputs.report}} ${{steps.new.outputs.report}} -c '${{env.compare_statistics}}' |