ci: migrate to pixi (#1470)

.github/workflows/build.yaml

@@ -14,61 +14,54 @@ on:
 env:
   PYTHON_VERSION: "3.11"
 
+defaults:
+  run:
+    shell: bash -e {0}
+
 jobs:
+  pixi_lock:
+    name: Pixi lock
+    runs-on: ubuntu-latest
+    steps:
+      - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0
+
   conda_build:
-    name: Build Conda Packages
-    runs-on: 'ubuntu-latest'
-    defaults:
-      run:
-        shell: bash -l {0}
-    env:
-      MPLBACKEND: "Agg"
+    name: Build Conda
+    needs: [pixi_lock]
+    runs-on: "ubuntu-latest"
     steps:
-      - uses: actions/checkout@v4
-      - name: Fetch unshallow
-        run: git fetch --prune --tags --unshallow -f
-      - uses: conda-incubator/setup-miniconda@v3
+      - uses: holoviz-dev/holoviz_tasks/pixi_install@v0
         with:
-          miniconda-version: "latest"
-          python-version: ${{ env.PYTHON_VERSION }}
-          auto-update-conda: true
-      - name: Set output
-        id: vars
-        run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
-      - name: conda setup
-        run: conda install anaconda-client conda-build setuptools_scm
+          environments: "build"
+          download-data: false
+          install: false
       - name: conda build
-        # TODO: remove --no-test when Dask 2024.4.2 is released on defaults
-        run: VERSION=`python -m setuptools_scm` conda build conda.recipe/ --no-test
+        run: pixi run -e build build-conda
+      - name: Set environment variables
+        run: |
+          echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+          echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV
       - name: conda dev upload
-        if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        run: anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $(VERSION=`python -m setuptools_scm` conda build --output conda.recipe)
+        if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')
+        run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE
       - name: conda main upload
-        if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc')))
-        run: anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $(VERSION=`python -m setuptools_scm` conda build --output conda.recipe)
+        if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc')))
+        run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE
+
   pip_build:
-    name: Build PyPI Packages
-    runs-on: 'ubuntu-latest'
-    defaults:
-      run:
-        shell: bash -l {0}
-    env:
-      MPLBACKEND: "Agg"
+    name: Build PyPI
+    needs: [pixi_lock]
+    runs-on: "ubuntu-latest"
+    permissions:
+      id-token: write
     steps:
-      - uses: actions/checkout@v4
-      - name: Fetch unshallow
-        run: git fetch --prune --tags --unshallow -f
-      - uses: actions/setup-python@v5
+      - uses: holoviz-dev/holoviz_tasks/pixi_install@v0
         with:
-          python-version: ${{ env.PYTHON_VERSION }}
-      - name: env setup
-        run: python -m pip install build
-      - name: pip build
-        run: python -m build
-      - name: Publish package to PyPI
+          environments: "build"
+          download-data: false
+          install: false
+      - name: Build package
+        run: pixi run -e build build-pip
+      - name: Publish to PyPI
         if: github.event_name == 'push'
         uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          user: ${{ secrets.PPU }}
-          password: ${{ secrets.PPP }}
-          packages-dir: dist/

.github/workflows/docs.yaml

@@ -20,45 +20,37 @@ on:
   schedule:
     - cron: '0 15 * * SUN'
 
+defaults:
+  run:
+    shell: bash -e {0}
+
 env:
   PYTHON_VERSION: "3.11"
 
 jobs:
-  conda_build_docs:
-    name: Documentation:conda
+  pixi_lock:
+    name: Pixi lock
+    runs-on: ubuntu-latest
+    steps:
+      - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0
+
+  docs_build:
+    name: Build Documentation
     runs-on: 'ubuntu-latest'
     timeout-minutes: 120
-    defaults:
-      run:
-        shell: bash -l {0}
     env:
-      DESC: "Documentation build"
-      MPLBACKEND: "Agg"
-      MOZ_HEADLESS: 1
       DISPLAY: ":99.0"
     steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v3
+      - uses: holoviz-dev/holoviz_tasks/pixi_install@v0
         with:
-          auto-update-conda: true
-          environment-file: envs/py${{ env.PYTHON_VERSION }}-docs.yaml
-          activate-environment: hvplotdocs
+          environments: docs
+      - name: Build documentation
+        run: pixi run -e docs docs-build
       - name: Set and echo git ref
         id: vars
         run: |
           echo "Deploying from ref ${GITHUB_REF#refs/*/}"
           echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
-      - name: install dev
-        # To make sure the latest is installed.
-        run: pip install -e . --no-deps
-      - name: conda info
-        run: conda info
-      - name: conda list
-        run: conda list
-      - name: build docs
-        run: sphinx-build -b html doc builtdocs
       - name: report failure
         if: failure()
         run: cat /tmp/sphinx-*.log | tail -n 100
@@ -84,13 +76,11 @@ jobs:
           cname: hvplot.holoviz.org
           force_orphan: true
           exclude_assets: '.doctrees'
+
   pip_build_docs:
-    name: Documentation:pip
+    name: Build Documentation (pip)
     runs-on: 'ubuntu-latest'
     timeout-minutes: 120
-    defaults:
-      run:
-        shell: bash -l {0}
     env:
       DESC: "Documentation build"
       MPLBACKEND: "Agg"

.github/workflows/nightly_lock.yaml

@@ -0,0 +1,25 @@
+name: nightly_lock
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *"
+
+env:
+  PACKAGE: "holoviews"
+
+jobs:
+  pixi_lock:
+    name: Pixi lock
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0
+      - name: Upload lock-file to S3
+        if: "!github.event.pull_request.head.repo.fork"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: "eu-west-1"
+        run: |
+          zip $(date +%Y-%m-%d).zip pixi.lock pixi.toml
+          aws s3 cp ./$(date +%Y-%m-%d).zip s3://assets.holoviz.org/lock/$PACKAGE/

.github/workflows/test.yaml

@@ -30,12 +30,17 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash -e {0}
+
 jobs:
   pre_commit:
     name: Run pre-commit
     runs-on: 'ubuntu-latest'
     steps:
-      - uses: holoviz-dev/holoviz_tasks/[email protected]
+      - uses: holoviz-dev/holoviz_tasks/pre-commit@v0
+
   setup:
     name: Setup workflow
     runs-on: ubuntu-latest
@@ -60,10 +65,10 @@ jobs:
         run: |
           MATRIX=$(jq -nsc '{
               "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
-              "python-version": ["3.9", "3.12"],
+              "environment": ["test-39", "test-312"],
               "exclude": [
                 {
-                  "python-version": "3.9",
+                  "environment": "test-39",
                   "os": "macos-latest"
                 }
               ]
@@ -74,24 +79,24 @@ jobs:
         run: |
           MATRIX=$(jq -nsc '{
               "os": ["ubuntu-latest", "macos-latest", "windows-latest"],
-              "python-version": ["3.9", "3.12"],
+              "environment": ["test-39", "test-312"],
               "include": [
                 {
-                  "python-version": "3.9",
+                  "environment": "test-39",
                   "os": "ubuntu-latest"
                 },
                 {
-                  "python-version": "3.10",
+                  "environment": "test-310",
                   "os": "ubuntu-latest"
                 },
                 {
-                  "python-version": "3.11",
+                  "environment": "test-311",
                   "os": "ubuntu-latest"
                 }
               ],
               "exclude": [
                 {
-                  "python-version": "3.9",
+                  "environment": "test-39",
                   "os": "macos-latest"
                 }
               ]
@@ -102,71 +107,82 @@ jobs:
         run: |
           MATRIX=$(jq -nsc '{
               "os": ["ubuntu-latest"],
-              "python-version": ["3.12"]
+              "environment": ["test-312"]
           }')
           echo "MATRIX=$MATRIX" >> $GITHUB_ENV
 
-  conda_suite:
-    name: conda tests:${{ matrix.os }}:${{ matrix.python-version }}
-    needs: [pre_commit, setup]
-    if: needs.setup.outputs.matrix_option != 'default'
+  pixi_lock:
+    name: Pixi lock
+    runs-on: ubuntu-latest
+    steps:
+      - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0
+        with:
+          cache: ${{ github.event.inputs.cache == 'true' || github.event.inputs.cache == '' }}
+
+  unit_test_suite:
+    name: unit:${{ matrix.environment }}:${{ matrix.os }}
+    needs: [pre_commit, setup, pixi_lock]
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
     timeout-minutes: 90
-    defaults:
-      run:
-        shell: bash -el {0}
+
     steps:
-      - uses: actions/checkout@v4
+      - uses: holoviz-dev/holoviz_tasks/pixi_install@v0
         with:
-          fetch-depth: 0
-      - uses: conda-incubator/setup-miniconda@v3
+          environments: ${{ matrix.environment }}
+      - name: Test Unit
+        run: pixi run -e ${{ matrix.environment }} test-unit-cov
+      - name: Test Geo
+        run: pixi run -e ${{ matrix.environment }} test-unit-geo-cov
+      - name: Test Examples
+        run: pixi run -e ${{ matrix.environment }} test-example
+      - uses: codecov/codecov-action@v4
         with:
-          auto-update-conda: true
-          environment-file: envs/py${{ matrix.python-version }}-tests.yaml
-          activate-environment: hvplottests
-      - name: conda info
-        run: conda info
-      - name: conda list
-        run: conda list
-      - name: bokeh sampledata
-        if: ${{ matrix.python-version == '3.9'}}
-        run: bokeh sampledata
-      - name: unit tests
-        run: pytest -v hvplot --cov=hvplot --cov-append
-      - name: unit tests geo
-        run: pytest -v hvplot --geo --cov=hvplot --cov-append
-      - name: examples tests
-        run: pytest -n logical --dist loadscope --nbval-lax -p no:python
+          token: ${{ secrets.CODECOV_TOKEN }}
+
+  core_test_suite:
+    name: core:${{ matrix.environment }}:${{ matrix.os }}
+    needs: [pre_commit, setup, pixi_lock]
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ["ubuntu-latest"]
+        environment: ["test-core"]
+    timeout-minutes: 120
+    steps:
+      - uses: holoviz-dev/holoviz_tasks/pixi_install@v0
+        with:
+          environments: ${{ matrix.environment }}
+      - name: Test Unit
+        run: pixi run -e ${{ matrix.environment }} test-unit
+
   pip_test:
-    name: pip tests:${{ matrix.os }}:${{ matrix.python-version }}
+    name: pip tests:${{ matrix.os }}:${{ matrix.environment }}
     needs: [pre_commit, setup]
+    if: needs.setup.outputs.matrix_option != 'default'
     timeout-minutes: 90
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.setup.outputs.matrix) }}
-    defaults:
-      run:
-        shell: bash -e {0}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: actions/setup-python@v5
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: '3.12'
       - name: install with geo
         run: python -m pip install -v --prefer-binary -e '.[tests, examples-tests, geo, hvdev, hvdev-geo, dev-extras]'
       - name: python version and pip list
         run: |
           python --version --version
           python -m pip list
-      - name: bokeh sampledata
-        if: ${{ matrix.python-version == '3.9'}}
-        run: bokeh sampledata
+      - name: download datasets
+        run: python scripts/download_data.py
       - name: unit tests
         run: pytest -v hvplot --cov=hvplot --cov-append
       - name: unit tests geo
@@ -181,3 +197,13 @@ jobs:
           verbose: false
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  result_test_suite:
+    name: result:test
+    needs: [unit_test_suite, core_test_suite, pip_test]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: check for failures
+        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+        run: echo job failed && exit 1