From a5e3e1e6eb9521e82b3af63409d8c73d9f0634d2 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:23:37 -0500 Subject: [PATCH] ADAP-1178: Add scheduled testing workflow (#607) --- .github/workflows/_integration-tests.yml | 4 +- .github/workflows/pull-request-checks.yml | 24 ++---- .github/workflows/scheduled-tests.yml | 98 +++++++++++++++++++++++ 3 files changed, 108 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/scheduled-tests.yml diff --git a/.github/workflows/_integration-tests.yml b/.github/workflows/_integration-tests.yml index 2143e9495..ae7defad2 100644 --- a/.github/workflows/_integration-tests.yml +++ b/.github/workflows/_integration-tests.yml @@ -129,7 +129,7 @@ jobs: integration-tests-bigquery-flaky: # we only run this for one python version to avoid running in parallel - if: ${{ inputs.package == 'dbt-bigquery' && inputs.python-version == '3.9' }} + if: ${{ inputs.package == 'dbt-bigquery' && inputs.python-version == vars.DEFAULT_PYTHON_VERSION }} runs-on: ${{ inputs.os }} environment: name: "dbt-bigquery" @@ -290,7 +290,7 @@ jobs: integration-tests-redshift-flaky: # we only run this for one python version to avoid running in parallel - if: ${{ inputs.package == 'dbt-redshift' && inputs.python-version == '3.9' }} + if: ${{ inputs.package == 'dbt-redshift' && inputs.python-version == vars.DEFAULT_PYTHON_VERSION }} runs-on: ${{ inputs.os }} environment: name: "dbt-redshift" diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 0ee4b6cbb..b4e224e45 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,5 +1,5 @@ name: "Pull request checks" -run-name: "Publish - #${{ github.event.number }} - ${{ github.actor }}" +run-name: "Pull request checks - #${{ github.event.number }} - ${{ github.actor }}" on: pull_request_target: @@ -52,10 +52,8 @@ jobs: fail-fast: false matrix: package: ${{ fromJSON(needs.affected-packages.outputs.changelog-entry-check) }} - os: [ubuntu-22.04] - python-version: ["3.9", "3.10", "3.11", "3.12"] with: - package: "dbt-athena" + package: ${{ matrix.package }} pull-request: ${{ github.event.pull_request.number }} secrets: inherit @@ -73,14 +71,12 @@ jobs: fail-fast: false matrix: package: ${{ fromJSON(needs.affected-packages.outputs.verify-build) }} - os: [ubuntu-22.04] - python-version: ["3.9", "3.10", "3.11", "3.12"] with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} + os: ${{ vars.DEFAULT_RUNNER }} + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} unit-tests: uses: ./.github/workflows/_unit-tests.yml @@ -90,14 +86,12 @@ jobs: fail-fast: false matrix: package: ${{ fromJSON(needs.affected-packages.outputs.unit-tests) }} - os: [ ubuntu-22.04 ] - python-version: ["3.9", "3.10", "3.11", "3.12"] with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} + os: ${{ vars.DEFAULT_RUNNER }} + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} integration-tests: uses: ./.github/workflows/_integration-tests.yml @@ -107,14 +101,12 @@ jobs: fail-fast: false matrix: package: ${{ fromJSON(needs.affected-packages.outputs.integration-tests) }} - os: [ubuntu-22.04] - python-version: ["3.9", "3.10", "3.11", "3.12"] with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - os: ${{ matrix.os }} - python-version: ${{ matrix.python-version }} + os: ${{ vars.DEFAULT_RUNNER }} + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} secrets: inherit # This job does nothing and is only used for branch protection diff --git a/.github/workflows/scheduled-tests.yml b/.github/workflows/scheduled-tests.yml new file mode 100644 index 000000000..4746f3e34 --- /dev/null +++ b/.github/workflows/scheduled-tests.yml @@ -0,0 +1,98 @@ +name: "Scheduled tests" + +on: + schedule: + - cron: '0 3 * * *' + workflow_dispatch: + +jobs: + verify-builds: + uses: ./.github/workflows/_verify-build.yml + strategy: + fail-fast: false + matrix: + package: + - "dbt-adapters" + - "dbt-tests-adapter" + - "dbt-athena" + - "dbt-athena-community" + - "dbt-bigquery" + - "dbt-postgres" + - "dbt-redshift" + - "dbt-snowflake" + - "dbt-spark" + os: [ubuntu-22.04, macos-14, windows-2022] + python-version: ${{ fromJson(vars.SUPPORTED_PYTHON_VERSIONS) }} + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + unit-tests: + uses: ./.github/workflows/_unit-tests.yml + strategy: + fail-fast: false + matrix: + package: + - "dbt-adapters" + - "dbt-athena" + - "dbt-athena-community" + - "dbt-bigquery" + - "dbt-postgres" + - "dbt-redshift" + - "dbt-snowflake" + - "dbt-spark" + os: [ubuntu-22.04, macos-14, windows-2022] + python-version: ${{ fromJson(vars.SUPPORTED_PYTHON_VERSIONS) }} + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + integration-tests: + uses: ./.github/workflows/_integration-tests.yml + strategy: + fail-fast: false + matrix: + package: + - "dbt-athena" + - "dbt-athena-community" + - "dbt-bigquery" + - "dbt-postgres" + - "dbt-redshift" + - "dbt-snowflake" + - "dbt-spark" + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + os: ${{ vars.DEFAULT_RUNNER }} + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + secrets: inherit + + notification: + if: always() + needs: [verify-builds, unit-tests, integration-tests] + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + # This step is needed to aggregate the results of the other jobs and apply it to this job for the Slack notification + - id: results + uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} + - uses: ravsamhq/notify-slack-action@v2 + if: ${{ always() }} + with: + token: ${{ secrets.GITHUB_TOKEN }} + status: ${{ job.status }} + notification_title: "{workflow} has {status_message}" + message_format: "{emoji} *{workflow}* {status_message} in <{repo_url}|{repo}>" + footer: "View run: {workflow_url}" + mention_users: "S066LNHS2N6" + mention_users_when: "failure,warnings" + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }}