performance-regression.yml

name: Performance Regression

on:
  pull_request:
    paths:
      # Workflow will only run if something in this path is changed
      - 'amazon/**'

env:
  report_statistics: 'file_size,time_mean,time_error,ops/s_mean,ops/s_error,memory_usage_peak'
  compare_statistics: 'file_size,time_mean'
  data_size: '100'
  spec_defaults: '{warmups:100,iterations:100}'
  specs: '{command:read,format:ion_text} {command:write,format:ion_text} {command:read,format:ion_binary} {command:write,format:ion_binary}'
  test_data_id: 'generated-test-data'
  run_cli: 'python amazon/ionbenchmark/ion_benchmark_cli.py'


jobs:
  generate-test-data:
    name: Generate Data
    runs-on: ubuntu-latest
    steps:
      # Generates data used for benchmarking
      - name: Checkout ion-data-generator
        uses: actions/checkout@v4
        with:
          repository: amazon-ion/ion-data-generator
          ref: main
      - name: Build ion-data-generator
        run: mvn clean install
      - name: Generate test Ion Data
        env:
          jar_file: target/ion-data-generator-1.0-SNAPSHOT.jar
          schema_dir: tst/com/amazon/ion/workflow
        run: |
          mkdir -p testData
          for test in nestedStruct nestedList sexp realWorldDataSchema01 realWorldDataSchema02 realWorldDataSchema03
          do
            java -jar $jar_file generate -S ${{env.data_size}} --input-ion-schema $schema_dir/${test}.isl testData/${test}.10n
          done
      - name: Upload test Ion Data to artifacts
        uses: actions/upload-artifact@v4
        with:
          name: ${{env.test_data_id}}
          path: testData

  prepopulate-pip-cache:
    # Since all the "Check" jobs can run in parallel, caching _could_ have basically no effect. In order to speed things
    # up, this step can run in parallel with the "Generate Data" job, pre-caching all the dependencies.
    name: Setup PIP Cache
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'
          cache: 'pip'
          cache-dependency-path: '**/requirements.txt'
      - run: |
          pip install -r requirements.txt
          # TODO: See if there's a way to cache the ion-c build output if it hasn't changed

  detect-regression:
    name: Check
    runs-on: ubuntu-latest
    needs: [generate-test-data, prepopulate-pip-cache]
    strategy:
      matrix:
        python-version: ['3.9', '3.11', 'pypy-3.8', 'pypy-3.10']
        test-data: ['nestedStruct', 'nestedList', 'sexp', 'realWorldDataSchema01', 'realWorldDataSchema02', 'realWorldDataSchema03']
      fail-fast: false
    steps:
      - name: Checkout the base of the PR
        uses: actions/checkout@v4
        with:
          ref: ${{ github.base_ref }}
          submodules: recursive
          path: baseline

      - name: Checkout the head of the PR
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.head_ref }}
          submodules: recursive
          path: new

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
          cache-dependency-path: '**/requirements.txt'

      - name: Download Test Data
        id: 'download'
        uses: actions/download-artifact@v4
        with:
          name: ${{env.test_data_id}}

      # Generates performance results for the previous commit
      - name: Create a virtual environment
        working-directory: ./baseline
        run: |
          pip install -r requirements.txt
          pip install .
      - name: Run baseline performance benchmark
        id: 'baseline'
        working-directory: ./baseline
        run: |
          ${{env.run_cli}} spec '${{env.specs}}' -d '${{env.spec_defaults}}' \
              -O '{input_file:"${{steps.download.outputs.download-path}}/${{ matrix.test-data }}.10n"}' \
              -o "$PWD/report.ion" -r '${{env.report_statistics}}'
          echo "::group::Ion Report"
          echo "$(<report.ion)"
          echo "::endgroup::"
          echo "report=$PWD/report.ion" >> "$GITHUB_OUTPUT"

      # Generates performance results for the current commit
      - name: Create a virtual environment and setup the package
        working-directory: ./new
        run: |
          pip install -r requirements.txt
          pip install .
      - name: Run new performance benchmark
        id: 'new'
        working-directory: ./new
        run: |
          ${{env.run_cli}} spec '${{env.specs}}' -d '${{env.spec_defaults}}' \
              -O '{input_file:"${{steps.download.outputs.download-path}}/${{ matrix.test-data }}.10n"}' \
              -o "$PWD/report.ion" -r '${{env.report_statistics}}'
          echo "::group::Ion Report"
          echo "$(<report.ion)"
          echo "::endgroup::"
          echo "report=$PWD/report.ion" >> "$GITHUB_OUTPUT"

      # Compare results and identify regression
      - name: Detect performance regression
        working-directory: ./new
        run: ${{env.run_cli}} compare --fail ${{steps.baseline.outputs.report}} ${{steps.new.outputs.report}} -c '${{env.compare_statistics}}'