Refactor to be able to completely disable os

saltstack · Nov 2, 2024 · af8a2ec · af8a2ec
1 parent f930ac3
commit af8a2ec
Show file tree

Hide file tree

Showing 30 changed files with 2,628 additions and 1,397 deletions.
diff --git a/.github/workflows/build-deps-ci-action-linux.yml b/.github/workflows/build-deps-ci-action-linux.yml
@@ -0,0 +1,174 @@
+---
+name: Install Test Dependencies
+
+on:
+  workflow_call:
+    inputs:
+      nox-session:
+        required: true
+        type: string
+        description: The nox session to run
+      salt-version:
+        type: string
+        required: true
+        description: The Salt version to set prior to running tests.
+      cache-prefix:
+        required: true
+        type: string
+        description: Seed used to invalidate caches
+      nox-version:
+        required: true
+        type: string
+        description: The nox version to install
+      nox-archive-hash:
+        required: true
+        type: string
+        description: Nox Tarball Cache Hash
+      python-version:
+        required: false
+        type: string
+        description: The python version to run tests with
+        default: "3.10"
+      package-name:
+        required: false
+        type: string
+        description: The onedir package name to use
+        default: salt
+
+
+env:
+  COLUMNS: 190
+  AWS_MAX_ATTEMPTS: "10"
+  AWS_RETRY_MODE: "adaptive"
+  PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
+  PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
+  PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
+
+jobs:
+
+  generate-matrix:
+    name: Generate Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
+    env:
+      PIP_INDEX_URL: https://pypi.org/simple
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: "Throttle Builds"
+        shell: bash
+        run: |
+          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"
+
+      - name: Checkout Source Code
+        uses: actions/checkout@v4
+
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-prefix }}
+        env:
+          PIP_INDEX_URL: https://pypi.org/simple
+
+      - name: Generate Test Matrix
+        id: generate-matrix
+        run: |
+          tools ci deps-matrix
+
+
+  linux-dependencies:
+    name: Linux
+    needs:
+      - generate-matrix
+    runs-on:
+      - linux-${{ matrix.arch }}
+    env:
+      USE_S3_CACHE: 'false'
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['linux'] }}
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: "Throttle Builds"
+        shell: bash
+        run: |
+          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"
+
+      - name: Checkout Source Code
+        uses: actions/checkout@v4
+
+      - name: Cache nox.linux.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
+        id: nox-dependencies-cache
+        uses: ./.github/actions/cache
+        with:
+          path: nox.linux.${{ matrix.arch }}.tar.*
+          key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|linux|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}
+
+      - name: Download Onedir Tarball as an Artifact
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz
+          path: artifacts/
+
+      - name: Decompress Onedir Tarball
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
+          cd artifacts
+          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz
+
+      - name: PyPi Proxy
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          sed -i '7s;^;--index-url=${{ vars.PIP_INDEX_URL }} --trusted-host ${{ vars.PIP_TRUSTED_HOST }} --extra-index-url=${{ vars.PIP_EXTRA_INDEX_URL }}\n;' requirements/static/ci/*/*.txt
+
+      - name: Setup Python Tools Scripts
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-prefix }}-build-deps-ci
+
+      - name: Install System Dependencies
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          echo true
+
+      - name: Install Nox
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          python3 -m pip install 'nox==${{ inputs.nox-version }}'
+
+      - name: Install Dependencies
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        env:
+          PRINT_TEST_SELECTION: "0"
+          PRINT_SYSTEM_INFO: "0"
+        run: |
+          nox --install-only -e ${{ inputs.nox-session }}
+
+      - name: Cleanup .nox Directory
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          nox --force-color -e "pre-archive-cleanup(pkg=False)"
+
+      - name: Compress .nox Directory
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          nox --force-color -e compress-dependencies -- linux ${{ matrix.arch }}
+
+      - name: Upload Nox Requirements Tarball
+        uses: actions/upload-artifact@v4
+        with:
+          name: nox-linux-${{ matrix.arch }}-${{ inputs.nox-session }}
+          path: nox.linux.${{ matrix.arch }}.tar.*
diff --git a/.github/workflows/build-deps-ci-action-macos.yml b/.github/workflows/build-deps-ci-action-macos.yml
@@ -0,0 +1,169 @@
+---
+name: Install Test Dependencies
+
+on:
+  workflow_call:
+    inputs:
+      nox-session:
+        required: true
+        type: string
+        description: The nox session to run
+      salt-version:
+        type: string
+        required: true
+        description: The Salt version to set prior to running tests.
+      cache-prefix:
+        required: true
+        type: string
+        description: Seed used to invalidate caches
+      nox-version:
+        required: true
+        type: string
+        description: The nox version to install
+      nox-archive-hash:
+        required: true
+        type: string
+        description: Nox Tarball Cache Hash
+      python-version:
+        required: false
+        type: string
+        description: The python version to run tests with
+        default: "3.10"
+      package-name:
+        required: false
+        type: string
+        description: The onedir package name to use
+        default: salt
+
+
+env:
+  COLUMNS: 190
+  AWS_MAX_ATTEMPTS: "10"
+  AWS_RETRY_MODE: "adaptive"
+  PIP_INDEX_URL: ${{ vars.PIP_INDEX_URL }}
+  PIP_TRUSTED_HOST: ${{ vars.PIP_TRUSTED_HOST }}
+  PIP_EXTRA_INDEX_URL: ${{ vars.PIP_EXTRA_INDEX_URL }}
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"
+
+jobs:
+
+  generate-matrix:
+    name: Generate Matrix
+    runs-on: ubuntu-latest
+    outputs:
+      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
+    env:
+      PIP_INDEX_URL: https://pypi.org/simple
+    steps:
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: "Throttle Builds"
+        shell: bash
+        run: |
+          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"
+
+      - name: Checkout Source Code
+        uses: actions/checkout@v4
+
+      - name: Setup Python Tools Scripts
+        uses: ./.github/actions/setup-python-tools-scripts
+        with:
+          cache-prefix: ${{ inputs.cache-prefix }}
+        env:
+          PIP_INDEX_URL: https://pypi.org/simple
+
+      - name: Generate Test Matrix
+        id: generate-matrix
+        run: |
+          tools ci deps-matrix
+
+
+  macos-dependencies:
+    name: MacOS
+    needs:
+      - generate-matrix
+    runs-on: ${{ matrix.distro-slug == 'macos-13-arm64' && 'macos-13-xlarge' || matrix.distro-slug }}
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['macos'] }}
+    env:
+      PIP_INDEX_URL: https://pypi.org/simple
+    steps:
+
+      - name: "Throttle Builds"
+        shell: bash
+        run: |
+          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"
+
+      - name: Checkout Source Code
+        uses: actions/checkout@v4
+
+      - name: Cache nox.macos.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
+        id: nox-dependencies-cache
+        uses: ./.github/actions/cache
+        with:
+          path: nox.macos.${{ matrix.arch }}.tar.*
+          key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|macos|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}
+
+      - name: Download Onedir Tarball as an Artifact
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz
+          path: artifacts/
+
+      - name: Decompress Onedir Tarball
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
+          cd artifacts
+          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz
+
+      - name: Set up Python ${{ inputs.python-version }}
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        uses: actions/setup-python@v5
+        with:
+          python-version: "${{ inputs.python-version }}"
+
+      - name: Install System Dependencies
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          brew install openssl@3
+
+      - name: Install Nox
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          python3 -m pip install 'nox==${{ inputs.nox-version }}'
+
+      - name: Install Dependencies
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        env:
+          PRINT_TEST_SELECTION: "0"
+          PRINT_SYSTEM_INFO: "0"
+        run: |
+          export PYCURL_SSL_LIBRARY=openssl
+          export LDFLAGS="-L/usr/local/opt/openssl@3/lib"
+          export CPPFLAGS="-I/usr/local/opt/openssl@3/include"
+          export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig"
+          nox --install-only -e ${{ inputs.nox-session }}
+
+      - name: Cleanup .nox Directory
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          nox --force-color -e "pre-archive-cleanup(pkg=False)"
+
+      - name: Compress .nox Directory
+        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
+        run: |
+          nox --force-color -e compress-dependencies -- macos ${{ matrix.arch }}
+
+      - name: Upload Nox Requirements Tarball
+        uses: actions/upload-artifact@v4
+        with:
+          name: nox-macos-${{ matrix.arch }}-${{ inputs.nox-session }}
+          path: nox.macos.${{ matrix.arch }}.tar.*