ci.yaml

name: CI

# yamllint disable-line rule:truthy
on:
  push:
    branches:
      - dev
      - rc
      - master
  pull_request: ~
  workflow_dispatch:
    inputs:
      full:
        description: "Full run (regardless of changes)"
        default: false
        type: boolean
      lint-only:
        description: "Skip pytest"
        default: false
        type: boolean
      pylint-only:
        description: "Only run pylint"
        default: false
        type: boolean
      mypy-only:
        description: "Only run mypy"
        default: false
        type: boolean

env:
  CACHE_VERSION: 3
  PIP_CACHE_VERSION: 3
  MYPY_CACHE_VERSION: 3
  HA_SHORT_VERSION: 2023.3
  DEFAULT_PYTHON: "3.10"
  ALL_PYTHON_VERSIONS: "['3.10']"
  PRE_COMMIT_CACHE: ~/.cache/pre-commit
  PIP_CACHE: /tmp/pip-cache
  SQLALCHEMY_WARN_20: 1
  PYTHONASYNCIODEBUG: 1
  HASS_CI: 1

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  info:
    name: Collect information & changes data
    outputs:
      # In case of issues with the partial run, use the following line instead:
      # test_full_suite: 'true'
      core: ${{ steps.core.outputs.changes }}
      integrations_glob: ${{ steps.info.outputs.integrations_glob }}
      integrations: ${{ steps.integrations.outputs.changes }}
      pre-commit_cache_key: ${{ steps.generate_pre-commit_cache_key.outputs.key }}
      python_cache_key: ${{ steps.generate_python_cache_key.outputs.key }}
      requirements: ${{ steps.core.outputs.requirements }}
      python_versions: ${{ steps.info.outputs.python_versions }}
      test_full_suite: ${{ steps.info.outputs.test_full_suite }}
      test_group_count: ${{ steps.info.outputs.test_group_count }}
      test_groups: ${{ steps.info.outputs.test_groups }}
      tests_glob: ${{ steps.info.outputs.tests_glob }}
      tests: ${{ steps.info.outputs.tests }}
    runs-on: ubuntu-20.04
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Generate partial Python venv restore key
        id: generate_python_cache_key
        run: >-
          echo "key=venv-${{ env.CACHE_VERSION }}-${{
            hashFiles('requirements_test.txt') }}-${{
            hashFiles('requirements_all.txt') }}-${{
            hashFiles('homeassistant/package_constraints.txt') }}" >> $GITHUB_OUTPUT
      - name: Generate partial pre-commit restore key
        id: generate_pre-commit_cache_key
        run: >-
          echo "key=pre-commit-${{ env.CACHE_VERSION }}-${{
            hashFiles('.pre-commit-config.yaml') }}"  >> $GITHUB_OUTPUT
      - name: Filter for core changes
        uses: dorny/paths-filter@v2.11.1
        id: core
        with:
          filters: .core_files.yaml
      - name: Create a list of integrations to filter for changes
        run: |
          integrations=$(ls -Ad ./homeassistant/components/[!_]*  | xargs -n 1 basename)
          touch .integration_paths.yaml
          for integration in $integrations; do
            echo "${integration}: [homeassistant/components/${integration}/**, tests/components/${integration}/**]" \
              >> .integration_paths.yaml;
          done
          echo "Result:"
          cat .integration_paths.yaml
      - name: Filter for integration changes
        uses: dorny/paths-filter@v2.11.1
        id: integrations
        with:
          filters: .integration_paths.yaml
      - name: Collect additional information
        id: info
        run: |
          # Defaults
          integrations_glob=""
          test_full_suite="true"
          test_groups="[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]"
          test_group_count=10
          tests="[]"
          tests_glob=""

          if [[ "${{ steps.integrations.outputs.changes }}" != "[]" ]];
          then
            # Create a file glob for the integrations
            integrations_glob=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '. | join(",")')
            [[ "${integrations_glob}" == *","* ]] && integrations_glob="{${integrations_glob}}"

            # Create list of testable integrations
            possible_integrations=$(echo '${{ steps.integrations.outputs.changes }}' | jq -cSr '.[]')
            tests=$(
              for integration in ${possible_integrations};
              do
                if [[ -d "tests/components/${integration}" ]]; then
                  echo -n "\"${integration}\",";
                fi;
              done
            )

            [[ ! -z "${tests}" ]] && tests="${tests::-1}"
            tests="[${tests}]"
            test_groups="${tests}"
            # Test group count should be 1, we don't split partial tests
            test_group_count=1

            # Create a file glob for the integrations tests
            tests_glob=$(echo "${tests}" | jq -cSr '. | join(",")')
            [[ "${tests_glob}" == *","* ]] && tests_glob="{${tests_glob}}"

            test_full_suite="false"
          fi

          # We need to run the full suite on certain branches.
          # Or, in case core files are touched, for the full suite as well.
          if [[ "${{ github.ref }}" == "refs/heads/dev" ]] \
            || [[ "${{ github.ref }}" == "refs/heads/master" ]] \
            || [[ "${{ github.ref }}" == "refs/heads/rc" ]] \
            || [[ "${{ steps.core.outputs.any }}" == "true" ]] \
            || [[ "${{ github.event.inputs.full }}" == "true" ]] \
            || [[ "${{ contains(github.event.pull_request.labels.*.name, 'ci-full-run') }}" == "true" ]];
          then
            test_groups="[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]"
            test_group_count=10
            test_full_suite="true"
          fi

          # Output & sent to GitHub Actions
          echo "python_versions: ${ALL_PYTHON_VERSIONS}"
          echo "python_versions=${ALL_PYTHON_VERSIONS}" >> $GITHUB_OUTPUT
          echo "test_full_suite: ${test_full_suite}"
          echo "test_full_suite=${test_full_suite}" >> $GITHUB_OUTPUT
          echo "integrations_glob: ${integrations_glob}"
          echo "integrations_glob=${integrations_glob}" >> $GITHUB_OUTPUT
          echo "test_group_count: ${test_group_count}"
          echo "test_group_count=${test_group_count}" >> $GITHUB_OUTPUT
          echo "test_groups: ${test_groups}"
          echo "test_groups=${test_groups}" >> $GITHUB_OUTPUT
          echo "tests: ${tests}"
          echo "tests=${tests}" >> $GITHUB_OUTPUT
          echo "tests_glob: ${tests_glob}"
          echo "tests_glob=${tests_glob}" >> $GITHUB_OUTPUT

  pre-commit:
    name: Prepare pre-commit base
    runs-on: ubuntu-20.04
    if: |
      github.event.inputs.pylint-only != 'true'
      && github.event.inputs.mypy-only != 'true'
    needs:
      - info
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Create Python virtual environment
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          python -m venv venv
          . venv/bin/activate
          python --version
          pip install "$(cat requirements_test.txt | grep pre-commit)"
      - name: Restore pre-commit environment from cache
        id: cache-precommit
        uses: actions/cache@v3.2.3
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Install pre-commit dependencies
        if: steps.cache-precommit.outputs.cache-hit != 'true'
        run: |
          . venv/bin/activate
          pre-commit install-hooks

  lint-black:
    name: Check black
    runs-on: ubuntu-20.04
    needs:
      - info
      - pre-commit
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.5.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-precommit
        uses: actions/cache/restore@v3.2.3
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-precommit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run black (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual black --all-files --show-diff-on-failure
      - name: Run black (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        shell: bash
        run: |
          . venv/bin/activate
          shopt -s globstar
          pre-commit run --hook-stage manual black --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*} --show-diff-on-failure

  lint-flake8:
    name: Check flake8
    runs-on: ubuntu-20.04
    needs:
      - info
      - pre-commit
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.5.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-precommit
        uses: actions/cache/restore@v3.2.3
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-precommit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Register flake8 problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/flake8.json"
      - name: Run flake8 (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual flake8 --all-files
      - name: Run flake8 (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        shell: bash
        run: |
          . venv/bin/activate
          shopt -s globstar
          pre-commit run --hook-stage manual flake8 --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*}
  lint-ruff:
    name: Check ruff
    runs-on: ubuntu-latest
    needs:
      - info
      - pre-commit
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.5.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-precommit
        uses: actions/cache/restore@v3.2.3
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-precommit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Register ruff problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/ruff.json"
      - name: Run ruff (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual ruff --all-files
      - name: Run ruff (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        shell: bash
        run: |
          . venv/bin/activate
          shopt -s globstar
          pre-commit run --hook-stage manual ruff --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*}
  lint-isort:
    name: Check isort
    runs-on: ubuntu-20.04
    needs:
      - info
      - pre-commit
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.5.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-precommit
        uses: actions/cache/restore@v3.2.3
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-precommit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1
      - name: Run isort
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual isort --all-files --show-diff-on-failure

  lint-other:
    name: Check other linters
    runs-on: ubuntu-20.04
    needs:
      - info
      - pre-commit
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        uses: actions/setup-python@v4.5.0
        id: python
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-venv-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore pre-commit environment from cache
        id: cache-precommit
        uses: actions/cache/restore@v3.2.3
        with:
          path: ${{ env.PRE_COMMIT_CACHE }}
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
              needs.info.outputs.pre-commit_cache_key }}
      - name: Fail job if pre-commit cache restore failed
        if: steps.cache-precommit.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore pre-commit environment from cache"
          exit 1

      - name: Run pyupgrade (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual pyupgrade --all-files --show-diff-on-failure
      - name: Run pyupgrade (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        shell: bash
        run: |
          . venv/bin/activate
          shopt -s globstar
          pre-commit run --hook-stage manual pyupgrade --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*} --show-diff-on-failure

      - name: Register yamllint problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/yamllint.json"
      - name: Run yamllint
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual yamllint --all-files --show-diff-on-failure

      - name: Register check-json problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/check-json.json"
      - name: Run check-json
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual check-json --all-files

      - name: Run prettier (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual prettier --all-files

      - name: Run prettier (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        shell: bash
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual prettier --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*}

      - name: Register check executables problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/check-executables-have-shebangs.json"
      - name: Run executables check
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual check-executables-have-shebangs --all-files

      - name: Register codespell problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/codespell.json"
      - name: Run codespell
        run: |
          . venv/bin/activate
          pre-commit run --show-diff-on-failure --hook-stage manual codespell --all-files

      - name: Register hadolint problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/hadolint.json"
      - name: Check Dockerfile
        uses: docker://hadolint/hadolint:v1.18.2
        with:
          args: hadolint Dockerfile
      - name: Check Dockerfile.dev
        uses: docker://hadolint/hadolint:v1.18.2
        with:
          args: hadolint Dockerfile.dev

      - name: Run bandit (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        run: |
          . venv/bin/activate
          pre-commit run --hook-stage manual bandit --all-files --show-diff-on-failure
      - name: Run bandit (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        shell: bash
        run: |
          . venv/bin/activate
          shopt -s globstar
          pre-commit run --hook-stage manual bandit --files {homeassistant,tests}/components/${{ needs.info.outputs.integrations_glob }}/{*,**/*} --show-diff-on-failure

  base:
    name: Prepare dependencies
    runs-on: ubuntu-20.04
    needs: info
    timeout-minutes: 60
    strategy:
      matrix:
        python-version: ${{ fromJSON(needs.info.outputs.python_versions) }}
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ matrix.python-version }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ matrix.python-version }}
          check-latest: true
      - name: Generate partial pip restore key
        id: generate-pip-key
        run: >-
          echo "key=pip-${{ env.PIP_CACHE_VERSION }}-${{
            env.HA_SHORT_VERSION }}-$(date -u '+%Y-%m-%dT%H:%M:%s')" >> $GITHUB_OUTPUT
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            needs.info.outputs.python_cache_key }}
      - name: Restore pip wheel cache
        if: steps.cache-venv.outputs.cache-hit != 'true'
        uses: actions/cache@v3.2.3
        with:
          path: ${{ env.PIP_CACHE }}
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            steps.generate-pip-key.outputs.key }}
          restore-keys: |
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-pip-${{ env.PIP_CACHE_VERSION }}-${{ env.HA_SHORT_VERSION }}-
      - name: Install additional OS dependencies
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          sudo apt-get update
          sudo apt-get -y install \
            bluez \
            ffmpeg \
            libavcodec-dev \
            libavdevice-dev \
            libavfilter-dev \
            libavformat-dev \
            libavutil-dev \
            libswresample-dev \
            libswscale-dev \
            libudev-dev
      - name: Create Python virtual environment
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          python -m venv venv
          . venv/bin/activate
          python --version
          pip install --cache-dir=$PIP_CACHE -U "pip>=21.0,<22.4" setuptools wheel
          pip install --cache-dir=$PIP_CACHE -r requirements_all.txt --use-deprecated=legacy-resolver
          pip install --cache-dir=$PIP_CACHE -r requirements_test.txt --use-deprecated=legacy-resolver
          pip install -e .

  hassfest:
    name: Check hassfest
    runs-on: ubuntu-20.04
    if: |
      github.event.inputs.pylint-only != 'true'
      && github.event.inputs.mypy-only != 'true'
    needs:
      - info
      - base
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore full Python ${{ env.DEFAULT_PYTHON }} virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            needs.info.outputs.python_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Run hassfest
        run: |
          . venv/bin/activate
          python -m script.hassfest --requirements --action validate

  gen-requirements-all:
    name: Check all requirements
    runs-on: ubuntu-20.04
    if: |
      github.event.inputs.pylint-only != 'true'
      && github.event.inputs.mypy-only != 'true'
    needs:
      - info
      - base
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore base Python virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            needs.info.outputs.python_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Run gen_requirements_all.py
        run: |
          . venv/bin/activate
          python -m script.gen_requirements_all validate

  pylint:
    name: Check pylint
    runs-on: ubuntu-20.04
    timeout-minutes: 20
    if: |
      github.event.inputs.mypy-only != 'true'
      || github.event.inputs.pylint-only == 'true'
    needs:
      - info
      - base
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Restore full Python ${{ env.DEFAULT_PYTHON }} virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            needs.info.outputs.python_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Register pylint problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/pylint.json"
      - name: Run pylint (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        run: |
          . venv/bin/activate
          python --version
          pylint --ignore-missing-annotations=y homeassistant
      - name: Run pylint (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        shell: bash
        run: |
          . venv/bin/activate
          python --version
          pylint --ignore-missing-annotations=y homeassistant/components/${{ needs.info.outputs.integrations_glob }}

  mypy:
    name: Check mypy
    runs-on: ubuntu-20.04
    if: |
      github.event.inputs.pylint-only != 'true'
      || github.event.inputs.mypy-only == 'true'
    needs:
      - info
      - base
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ env.DEFAULT_PYTHON }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ env.DEFAULT_PYTHON }}
          check-latest: true
      - name: Generate partial mypy restore key
        id: generate-mypy-key
        run: |
          mypy_version=$(cat requirements_test.txt | grep mypy | cut -d '=' -f 3)
          echo "version=$mypy_version" >> $GITHUB_OUTPUT
          echo "key=mypy-${{ env.MYPY_CACHE_VERSION }}-$mypy_version-${{
            env.HA_SHORT_VERSION }}-$(date -u '+%Y-%m-%dT%H:%M:%s')" >> $GITHUB_OUTPUT
      - name: Restore full Python ${{ env.DEFAULT_PYTHON }} virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            needs.info.outputs.python_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Restore mypy cache
        uses: actions/cache@v3.2.3
        with:
          path: .mypy_cache
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            steps.generate-mypy-key.outputs.key }}
          restore-keys: |
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-mypy-${{
            env.MYPY_CACHE_VERSION }}-${{ steps.generate-mypy-key.outputs.version }}-${{
            env.HA_SHORT_VERSION }}-
      - name: Register mypy problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/mypy.json"
      - name: Run mypy (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        run: |
          . venv/bin/activate
          python --version
          mypy homeassistant pylint
      - name: Run mypy (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        shell: bash
        run: |
          . venv/bin/activate
          python --version
          mypy homeassistant/components/${{ needs.info.outputs.integrations_glob }}

  pip-check:
    runs-on: ubuntu-20.04
    if: |
      github.event.inputs.pylint-only != 'true'
      && github.event.inputs.mypy-only != 'true'
    needs:
      - info
      - base
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ fromJson(needs.info.outputs.python_versions) }}
    name: Run pip check ${{ matrix.python-version }}
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ matrix.python-version }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ matrix.python-version }}
          check-latest: true
      - name: Restore full Python ${{ matrix.python-version }} virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: >-
            ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            needs.info.outputs.python_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Run pip check
        run: |
          . venv/bin/activate
          ./script/pip_check $PIP_CACHE

  pytest:
    runs-on: ubuntu-20.04
    if: |
      (github.event_name != 'push' || github.event.repository.full_name == 'home-assistant/core')
      && github.event.inputs.lint-only != 'true'
      && github.event.inputs.pylint-only != 'true'
      && github.event.inputs.mypy-only != 'true'
      && (needs.info.outputs.test_full_suite == 'true' || needs.info.outputs.tests_glob)
    needs:
      - info
      - base
      - gen-requirements-all
      - hassfest
      - lint-black
      - lint-other
      - lint-isort
      - mypy
    strategy:
      fail-fast: false
      matrix:
        group: ${{ fromJson(needs.info.outputs.test_groups) }}
        python-version: ${{ fromJson(needs.info.outputs.python_versions) }}
    name: >-
      Run tests Python ${{ matrix.python-version }} (${{ matrix.group }})
    steps:
      - name: Install additional OS dependencies
        run: |
          sudo apt-get update
          sudo apt-get -y install \
            bluez \
            ffmpeg
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ matrix.python-version }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ matrix.python-version }}
          check-latest: true
      - name: Restore full Python ${{ matrix.python-version }} virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            needs.info.outputs.python_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Register Python problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/python.json"
      - name: Install Pytest Annotation plugin
        run: |
          . venv/bin/activate
          # Ideally this should be part of our dependencies
          # However this plugin is fairly new and doesn't run correctly
          # on a non-GitHub environment.
          pip install pytest-github-actions-annotate-failures==0.1.3
      - name: Register pytest slow test problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/pytest-slow.json"
      - name: Run pytest (fully)
        if: needs.info.outputs.test_full_suite == 'true'
        timeout-minutes: 60
        run: |
          . venv/bin/activate
          python --version
          python3 -X dev -m pytest \
            -qq \
            --timeout=9 \
            --durations=10 \
            -n auto \
            --dist=loadfile \
            --test-group-count ${{ needs.info.outputs.test_group_count }} \
            --test-group=${{ matrix.group }} \
            --cov="homeassistant" \
            --cov-report=xml \
            -o console_output_style=count \
            -p no:sugar \
            tests
      - name: Run pytest (partially)
        if: needs.info.outputs.test_full_suite == 'false'
        timeout-minutes: 10
        shell: bash
        run: |
          . venv/bin/activate
          python --version

          if [[ ! -f "tests/components/${{ matrix.group }}/__init__.py" ]]; then
            echo "::error:: missing file tests/components/${{ matrix.group }}/__init__.py"
            exit 1
          fi

          python3 -X dev -m pytest \
            -qq \
            --timeout=9 \
            -n auto \
            --cov="homeassistant.components.${{ matrix.group }}" \
            --cov-report=xml \
            --cov-report=term-missing \
            -o console_output_style=count \
            --durations=0 \
            --durations-min=1 \
            -p no:sugar \
            tests/components/${{ matrix.group }}
      - name: Upload coverage artifact
        uses: actions/upload-artifact@v3.1.2
        with:
          name: coverage-${{ matrix.python-version }}-${{ matrix.group }}
          path: coverage.xml
      - name: Check dirty
        run: |
          ./script/check_dirty

  pytest-mariadb:
    runs-on: ubuntu-20.04
    services:
      mariadb:
        image: mariadb:10.9.3
        ports:
          - 3306:3306
        env:
          MYSQL_ROOT_PASSWORD: password
        options: --health-cmd="mysqladmin ping -uroot -ppassword" --health-interval=5s --health-timeout=2s --health-retries=3
    if: |
      (github.event_name != 'push' || github.event.repository.full_name == 'home-assistant/core')
      && github.event.inputs.lint-only != 'true'
      && github.event.inputs.pylint-only != 'true'
      && github.event.inputs.mypy-only != 'true'
      && needs.info.outputs.test_full_suite == 'true'
    needs:
      - info
      - base
      - gen-requirements-all
      - hassfest
      - lint-black
      - lint-other
      - lint-isort
      - mypy
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ fromJson(needs.info.outputs.python_versions) }}
    name: >-
      Run tests Python ${{ matrix.python-version }} (mariadb)
    steps:
      - name: Install additional OS dependencies
        run: |
          sudo apt-get update
          sudo apt-get -y install \
            bluez \
            ffmpeg \
            libmariadb-dev-compat
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Set up Python ${{ matrix.python-version }}
        id: python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ matrix.python-version }}
          check-latest: true
      - name: Restore full Python ${{ matrix.python-version }} virtual environment
        id: cache-venv
        uses: actions/cache/restore@v3.2.3
        with:
          path: venv
          key: ${{ runner.os }}-${{ steps.python.outputs.python-version }}-${{
            needs.info.outputs.python_cache_key }}
      - name: Fail job if Python cache restore failed
        if: steps.cache-venv.outputs.cache-hit != 'true'
        run: |
          echo "Failed to restore Python virtual environment from cache"
          exit 1
      - name: Register Python problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/python.json"
      - name: Install Pytest Annotation plugin
        run: |
          . venv/bin/activate
          # Ideally this should be part of our dependencies
          # However this plugin is fairly new and doesn't run correctly
          # on a non-GitHub environment.
          pip install pytest-github-actions-annotate-failures==0.1.3
      - name: Register pytest slow test problem matcher
        run: |
          echo "::add-matcher::.github/workflows/matchers/pytest-slow.json"
      - name: Install SQL Python libraries
        run: |
          . venv/bin/activate
          pip install mysqlclient sqlalchemy_utils
      - name: Run pytest (partially)
        timeout-minutes: 15
        shell: bash
        run: |
          . venv/bin/activate
          python --version

          python3 -X dev -m pytest \
            -qq \
            --timeout=20 \
            -n 1 \
            --cov="homeassistant.components.recorder" \
            --cov-report=xml \
            --cov-report=term-missing \
            -o console_output_style=count \
            --durations=10 \
            -p no:sugar \
            --dburl=mysql://root:password@127.0.0.1/homeassistant-test \
            tests/components/recorder
      - name: Upload coverage artifact
        uses: actions/upload-artifact@v3.1.2
        with:
          name: coverage-${{ matrix.python-version }}-mariadb
          path: coverage.xml
      - name: Check dirty
        run: |
          ./script/check_dirty

  coverage:
    name: Upload test coverage to Codecov
    runs-on: ubuntu-20.04
    needs:
      - info
      - pytest
    steps:
      - name: Check out code from GitHub
        uses: actions/checkout@v3.3.0
      - name: Download all coverage artifacts
        uses: actions/download-artifact@v3
      - name: Upload coverage to Codecov (full coverage)
        if: needs.info.outputs.test_full_suite == 'true'
        uses: codecov/codecov-action@v3.1.1
        with:
          flags: full-suite
      - name: Upload coverage to Codecov (partial coverage)
        if: needs.info.outputs.test_full_suite == 'false'
        uses: codecov/codecov-action@v3.1.1