ci.yml

name: CI

permissions:
  contents: read
  id-token: write

on:
  workflow_call:
    inputs:
      ref:
        required: true
        description: "GitHub ref to use"
        type: string
      version:
        required: true
        description: "Version to produce"
        type: string
      build-all-targets:
        required: false
        default: true
        description: "Build targets to produce, false builds only for Linux amd64."
        type: boolean
      test-codegen:
        required: false
        default: true
        description: "Whether to run per-language codegen tests."
        type: boolean
      lint:
        required: false
        default: true
        description: "Whether to run lints"
        type: boolean
      test-version-sets:
        required: false
        default: minimum current
        description: Version sets on which to run integration tests
        type: string
      integration-test-platforms:
        required: false
        default: ubuntu-latest
        description: Platforms on which to run integration tests, as a space delimited list
        type: string
      acceptance-test-platforms:
        required: false
        default: windows-latest macos-latest
        description: Platforms on which to run integration tests, as a space delimited list
        type: string
      enable-coverage:
        description: "Collects coverage stats; requires cov-enabled builds"
        default: false
        required: false
        type: boolean
    secrets:
      PULUMI_BOT_TOKEN:
        required: true
        description: "GitHub access token, required to mitigate GitHub rate limits"
      PULUMI_PROD_ACCESS_TOKEN:
        required: false
        description: "Pulumi access token, required to run tests against the service"

jobs:
  matrix:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ inputs.ref }}
      - name: Configure Go Cache Key
        env:
          CACHE_KEY: "matrix-setup"
        run: echo "$CACHE_KEY" > .gocache.tmp
      - name: Setup Go Caching
        uses: actions/setup-go@v3 # only used by gotestsum
        with:
          go-version: '>=1.19.0' # decoupled from version sets, only used by gotestsum
          cache: true
          cache-dependency-path: |
            pkg/go.sum
            .gocache.tmp
      - uses: actions/checkout@v3
        with:
          repository: dnephin/gotestsum
          ref: d09768c81065b404caed0855eb3ab8f11a2a4431
          path: vendor/gotestsum
      - run: |
          cd vendor/gotestsum
          go install .
      - uses: actions/cache@v3
        with:
          path: test-results
          key: read-gotestsum-timing-${{ github.run_number }}
          restore-keys: gotestsum-timing-
      - name: build matrix
        id: matrix
        env:
          BUILD_ALL_TARGETS: ${{ inputs.build-all-targets }}
          TEST_CODEGEN: ${{ inputs.test-codegen }}
          TEST_VERSION_SETS: ${{ inputs.test-version-sets }}
          INPUT_INTEGRATION_TEST_PLATFORMS: ${{ inputs.integration-test-platforms }}
          INPUT_ACCEPTANCE_TEST_PLATFORMS: ${{ inputs.acceptance-test-platforms }}
        run: |
          echo "::group::Prime test timing data"
          mkdir -p test-results
          find test-results -type f -empty -print -delete || true
          echo "::endgroup::"

          echo "::group::Remove old test timing data"
          # Timing data prior to this date is unreliable. Codegen tests modified in #11052 and
          # merged Monday Oct 17 at 6PM Pacific.
          find test-results -type f ! -newermt "2022-10-17T18:00-07:00" -print -delete || true
          echo "::endgroup::"

          echo "::group::Test matrix variables"
          readarray -td' ' VERSION_SETS_TO_TEST < <(echo -n "$TEST_VERSION_SETS"); declare -p VERSION_SETS_TO_TEST;
          readarray -td' ' INTEGRATION_PLATFORMS < <(echo -n "$INPUT_INTEGRATION_TEST_PLATFORMS"); declare -p INTEGRATION_PLATFORMS;
          readarray -td' ' ACCEPTANCE_PLATFORMS < <(echo -n "$INPUT_ACCEPTANCE_TEST_PLATFORMS"); declare -p ACCEPTANCE_PLATFORMS;
          BUILD_TARGETS='[
            { "os": "linux",   "arch": "amd64", "build-platform": "ubuntu-latest" }
          ]'
          if [ "${BUILD_ALL_TARGETS}" = "true" ]; then
            BUILD_TARGETS='[
              { "os": "linux",   "arch": "amd64", "build-platform": "ubuntu-latest" },
              { "os": "linux",   "arch": "arm64", "build-platform": "ubuntu-latest" },
              { "os": "windows", "arch": "amd64", "build-platform": "ubuntu-latest" },
              { "os": "windows", "arch": "arm64", "build-platform": "ubuntu-latest" },
              { "os": "darwin",  "arch": "amd64", "build-platform": "ubuntu-latest" },
              { "os": "darwin",  "arch": "arm64", "build-platform": "ubuntu-latest" }
            ]'
          fi

          CODEGEN_TESTS_FLAG=--codegen-tests
          PKG_UNIT_TEST_PARTITIONS=7
          if [ "${TEST_CODEGEN}" = "false" ]; then
            CODEGEN_TESTS_FLAG=--no-codegen-tests
            PKG_UNIT_TEST_PARTITIONS=3
          fi

          UNIT_TEST_MATRIX=$(
            ./scripts/get-job-matrix.py \
            -vvv \
            generate-matrix \
            --kind unit-test \
            "$CODEGEN_TESTS_FLAG" \
            --platform ubuntu-latest \
            --version-set current \
            --partition-module pkg "$PKG_UNIT_TEST_PARTITIONS" \
            --partition-module sdk 1 \
            --partition-module sdk/go/pulumi-language-go 1 \
            --partition-module sdk/nodejs/cmd/pulumi-language-nodejs 1 \
            --partition-module sdk/python/cmd/pulumi-language-python 1 \
            --partition-module tests 2
          )

          INTEGRATION_TEST_MATRIX=$(
            ./scripts/get-job-matrix.py \
            -vvv \
            generate-matrix \
            --kind integration-test \
            "$CODEGEN_TESTS_FLAG" \
            --platform "${INTEGRATION_PLATFORMS[@]}" \
            --version-set "${VERSION_SETS_TO_TEST[@]}" \
            --partition-module pkg 1 \
            --partition-module sdk 1 \
            --partition-module tests 2 \
            --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8
          )

          ACCEPTANCE_TEST_MATRIX=$(
            ./scripts/get-job-matrix.py \
            -vvv \
            generate-matrix \
            --kind acceptance-test \
            "$CODEGEN_TESTS_FLAG" \
            --tags all xplatform_acceptance \
            --platform "${ACCEPTANCE_PLATFORMS[@]}" \
            --version-set current \
            --partition-module pkg 1 \
            --partition-module sdk 1 \
            --partition-module tests 1 \
            --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 2
          )
          echo "::endgroup::"

          echo "::group::Version set variable"
          VERSION_SET=$(./scripts/get-job-matrix.py \
            generate-version-set \
            --version-set current
          )
          echo "::endgroup::"

          echo "::group::Unit test matrix"
          echo "$UNIT_TEST_MATRIX" | yq -P '.'
          echo "::endgroup::"
          echo "::group::Integration test matrix"
          echo "$INTEGRATION_TEST_MATRIX" | yq -P '.'
          echo "::endgroup::"
          echo "::group::acceptance test matrix"
          echo "$ACCEPTANCE_TEST_MATRIX" | yq -P '.'
          echo "::endgroup::"
          echo "::group::Version set"
          echo "$VERSION_SET" | yq -P '.'
          echo "::endgroup::"

          echo "::group::Set outputs"
          ./.github/scripts/set-output unit-test-matrix "${UNIT_TEST_MATRIX}"
          ./.github/scripts/set-output integration-test-matrix "${INTEGRATION_TEST_MATRIX}"
          ./.github/scripts/set-output acceptance-test-matrix "${ACCEPTANCE_TEST_MATRIX}"
          ./.github/scripts/set-output version-set "${VERSION_SET}"
          ./.github/scripts/set-output build-targets "${BUILD_TARGETS}"
          echo "::endgroup::"
    outputs:
      unit-test-matrix: "${{ fromJson(steps.matrix.outputs.unit-test-matrix) }}"
      integration-test-matrix: "${{ fromJson(steps.matrix.outputs.integration-test-matrix) }}"
      acceptance-test-matrix: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix) }}"
      version-set: "${{ fromJson(steps.matrix.outputs.version-set) }}"
      build-targets: "${{ fromJson(steps.matrix.outputs.build-targets) }}"

  lint:
    name: Lint
    needs: [matrix]
    if: ${{ inputs.lint }}
    uses: ./.github/workflows/ci-lint.yml
    with:
      ref: ${{ inputs.ref }}
      version-set: ${{ needs.matrix.outputs.version-set }}

  build-binaries:
    name: build binaries
    needs: [matrix]
    strategy:
      # To avoid tying up macOS runners:
      # If using IDE, ignore yaml-schema error: 'Incorrect type. Expected "boolean"'
      fail-fast: ${{ contains(needs.matrix.outputs.build-targets, 'macos') }}
      matrix:
        target: ${{ fromJson(needs.matrix.outputs.build-targets) }}
    uses: ./.github/workflows/ci-build-binaries.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      os: ${{ matrix.target.os }}
      arch: ${{ matrix.target.arch }}
      build-platform: ${{ matrix.target.build-platform }}
      version-set: ${{ needs.matrix.outputs.version-set }}
      enable-coverage: ${{ inputs.enable-coverage }}
    secrets: inherit

  build-sdks:
    name: Build SDKs
    needs: [matrix]
    uses: ./.github/workflows/ci-build-sdks.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      version-set: ${{ needs.matrix.outputs.version-set }}
    secrets: inherit

  # Tests that can run concurrently with builds.
  unit-test:
    # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
    # appearing in the rendered title of the job name: It changes this:
    #   CI / Unit Test (cd sdk/dotnet && make dotnet_test, cd sdk/dotnet && make dotnet_test, macos-11, mi... / sdk/dotnet dotnet_test on macos-11/current
    #   (See: https://github.com/pulumi/pulumi/runs/8241055084?check_suite_focus=true#logs)
    # To this:
    #   CI / Unit Test / sdk/dotnet dotnet_test on macos-11/current
    name: Unit Test${{ matrix.platform && '' }}
    needs: [matrix, lint]
    if: ${{ needs.matrix.outputs.unit-test-matrix != '{}' }}
    strategy:
      fail-fast: ${{ contains(needs.matrix.outputs.unit-test-matrix, 'macos') }}
      matrix: ${{ fromJson(needs.matrix.outputs.unit-test-matrix) }}
    uses: ./.github/workflows/ci-run-test.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      platform: ${{ matrix.platform }}

      test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
      test-command: ${{ matrix.test-suite.command }}
      is-integration-test: false
      enable-coverage: ${{ inputs.enable-coverage }}
      # require-build: false # TODO, remove ${{ matrix.require-build || false }}

      version-set: ${{ toJson(matrix.version-set) }}
    secrets: inherit

  # Tests that depend on builds
  integration-test:
    # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
    # appearing in the rendered title of the job name. See: unit test.
    name: Integration Test${{ matrix.platform && '' }}
    needs: [matrix, build-binaries, build-sdks, lint]
    if: ${{ needs.matrix.outputs.integration-test-matrix != '{}' }}
    strategy:
      fail-fast: ${{ contains(needs.matrix.outputs.integration-test-matrix, 'macos') }}
      matrix: ${{ fromJson(needs.matrix.outputs.integration-test-matrix) }}
    uses: ./.github/workflows/ci-run-test.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      platform: ${{ matrix.platform }}

      test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
      test-command: ${{ matrix.test-suite.command }}
      is-integration-test: true # TODO: set to true here
      enable-coverage: ${{ inputs.enable-coverage }}
      # require-build: false # TODO, remove ${{ matrix.require-build || false }}

      version-set: ${{ toJson(matrix.version-set) }}
    secrets: inherit

  # Tests that depend on builds, but a smaller subset against Windows & MacOS platforms.
  acceptance-test:
    # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from
    # appearing in the rendered title of the job name. See: unit test.
    name: Acceptance Test${{ matrix.platform && '' }}
    needs: [matrix, build-binaries, build-sdks]
    if: ${{ needs.matrix.outputs.acceptance-test-matrix != '{}' }}
    # alow jobs to fail if the platform contains windows
    strategy:
      fail-fast: ${{ contains(needs.matrix.outputs.acceptance-test-matrix, 'macos') }}
      matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix) }}
    uses: ./.github/workflows/ci-run-test.yml
    with:
      ref: ${{ inputs.ref }}
      version: ${{ inputs.version }}
      platform: ${{ matrix.platform }}

      test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }}
      test-command: ${{ matrix.test-suite.command }}
      is-integration-test: true # TODO: set to true here
      enable-coverage: ${{ inputs.enable-coverage }}
      # require-build: false # TODO, remove ${{ matrix.require-build || false }}

      version-set: ${{ toJson(matrix.version-set) }}
    secrets: inherit

  test-collect-reports:
    needs: [unit-test, integration-test, acceptance-test]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/cache@v3
        with:
          path: test-results
          key: gotestsum-timing-${{ github.run_number }}
          restore-keys: gotestsum-timing-
      - uses: actions/download-artifact@v3
        continue-on-error: true
        with:
          name: gotestsum-test-results
          path: test-results
      - name: List and clean up test results
        continue-on-error: true
        run: |
          ls -lhR test-results
          find test-results -mindepth 1 -name '*.json' -mtime +7 -delete