From c8c53430bc5079e9be85436e27cf5184397c2a0c Mon Sep 17 00:00:00 2001 From: maximlt Date: Mon, 20 Jan 2025 20:25:44 +0100 Subject: [PATCH 01/14] migrate CI to pixi --- .github/workflows/build.yaml | 85 ++++--- .github/workflows/docs.yaml | 46 ++-- .github/workflows/nightly_lock.yaml | 25 +++ .github/workflows/test.yaml | 88 +++++--- .gitignore | 4 + pixi.toml | 208 ++++++++++++++++++ scripts/conda/build.sh | 14 ++ .../conda/recipe}/meta.yaml | 4 +- scripts/download_data.py | 17 ++ 9 files changed, 385 insertions(+), 106 deletions(-) create mode 100644 .github/workflows/nightly_lock.yaml create mode 100644 pixi.toml create mode 100644 scripts/conda/build.sh rename {conda.recipe => scripts/conda/recipe}/meta.yaml (89%) create mode 100644 scripts/download_data.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 46a9d8a09..e84d791ed 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -14,61 +14,54 @@ on: env: PYTHON_VERSION: "3.11" +defaults: + run: + shell: bash -e {0} + jobs: + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0 + conda_build: - name: Build Conda Packages - runs-on: 'ubuntu-latest' - defaults: - run: - shell: bash -l {0} - env: - MPLBACKEND: "Agg" + name: Build Conda + needs: [pixi_lock] + runs-on: "ubuntu-latest" steps: - - uses: actions/checkout@v4 - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - - uses: conda-incubator/setup-miniconda@v3 + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 with: - miniconda-version: "latest" - python-version: ${{ env.PYTHON_VERSION }} - auto-update-conda: true - - name: Set output - id: vars - run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - name: conda setup - run: conda install anaconda-client conda-build setuptools_scm + environments: "build" + download-data: false + install: false - name: conda build - # TODO: remove --no-test when Dask 2024.4.2 is released on defaults - run: VERSION=`python -m setuptools_scm` conda build conda.recipe/ --no-test + run: pixi run -e build build-conda + - name: Set environment variables + run: | + echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV - name: conda dev upload - if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) - run: anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $(VERSION=`python -m setuptools_scm` conda build --output conda.recipe) + if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') + run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE - name: conda main upload - if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) - run: anaconda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $(VERSION=`python -m setuptools_scm` conda build --output conda.recipe) + if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) + run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE + pip_build: - name: Build PyPI Packages - runs-on: 'ubuntu-latest' - defaults: - run: - shell: bash -l {0} - env: - MPLBACKEND: "Agg" + name: Build PyPI + needs: [pixi_lock] + runs-on: "ubuntu-latest" + permissions: + id-token: write steps: - - uses: actions/checkout@v4 - - name: Fetch unshallow - run: git fetch --prune --tags --unshallow -f - - uses: actions/setup-python@v5 + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 with: - python-version: ${{ env.PYTHON_VERSION }} - - name: env setup - run: python -m pip install build - - name: pip build - run: python -m build - - name: Publish package to PyPI + environments: "build" + download-data: false + install: false + - name: Build package + run: pixi run -e build build-pip + - name: Publish to PyPI if: github.event_name == 'push' uses: pypa/gh-action-pypi-publish@release/v1 - with: - user: ${{ secrets.PPU }} - password: ${{ secrets.PPP }} - packages-dir: dist/ diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 2f2999444..e0d55a82d 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -20,45 +20,37 @@ on: schedule: - cron: '0 15 * * SUN' +defaults: + run: + shell: bash -e {0} + env: PYTHON_VERSION: "3.11" jobs: - conda_build_docs: - name: Documentation:conda + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0 + + docs_build: + name: Build Documentation runs-on: 'ubuntu-latest' timeout-minutes: 120 - defaults: - run: - shell: bash -l {0} env: - DESC: "Documentation build" - MPLBACKEND: "Agg" - MOZ_HEADLESS: 1 DISPLAY: ":99.0" steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3 + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 with: - auto-update-conda: true - environment-file: envs/py${{ env.PYTHON_VERSION }}-docs.yaml - activate-environment: hvplotdocs + environments: docs + - name: Build documentation + run: pixi run -e docs docs-build - name: Set and echo git ref id: vars run: | echo "Deploying from ref ${GITHUB_REF#refs/*/}" echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - - name: install dev - # To make sure the latest is installed. - run: pip install -e . --no-deps - - name: conda info - run: conda info - - name: conda list - run: conda list - - name: build docs - run: sphinx-build -b html doc builtdocs - name: report failure if: failure() run: cat /tmp/sphinx-*.log | tail -n 100 @@ -84,13 +76,11 @@ jobs: cname: hvplot.holoviz.org force_orphan: true exclude_assets: '.doctrees' + pip_build_docs: - name: Documentation:pip + name: Build Documentation (pip) runs-on: 'ubuntu-latest' timeout-minutes: 120 - defaults: - run: - shell: bash -l {0} env: DESC: "Documentation build" MPLBACKEND: "Agg" diff --git a/.github/workflows/nightly_lock.yaml b/.github/workflows/nightly_lock.yaml new file mode 100644 index 000000000..00e670c88 --- /dev/null +++ b/.github/workflows/nightly_lock.yaml @@ -0,0 +1,25 @@ +name: nightly_lock +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * *" + +env: + PACKAGE: "holoviews" + +jobs: + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0 + - name: Upload lock-file to S3 + if: "!github.event.pull_request.head.repo.fork" + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "eu-west-1" + run: | + zip $(date +%Y-%m-%d).zip pixi.lock pixi.toml + aws s3 cp ./$(date +%Y-%m-%d).zip s3://assets.holoviz.org/lock/$PACKAGE/ diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2d80995db..f695d1585 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -30,12 +30,17 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash -e {0} + jobs: pre_commit: name: Run pre-commit runs-on: 'ubuntu-latest' steps: - - uses: holoviz-dev/holoviz_tasks/pre-commit@v0.1a19 + - uses: holoviz-dev/holoviz_tasks/pre-commit@v0 + setup: name: Setup workflow runs-on: ubuntu-latest @@ -106,51 +111,64 @@ jobs: }') echo "MATRIX=$MATRIX" >> $GITHUB_ENV - conda_suite: - name: conda tests:${{ matrix.os }}:${{ matrix.python-version }} - needs: [pre_commit, setup] - if: needs.setup.outputs.matrix_option != 'default' + pixi_lock: + name: Pixi lock + runs-on: ubuntu-latest + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_lock@v0 + with: + cache: ${{ github.event.inputs.cache == 'true' || github.event.inputs.cache == '' }} + + unit_test_suite: + name: unit:${{ matrix.environment }}:${{ matrix.os }} + needs: [pre_commit, setup, pixi_lock] runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: ${{ fromJson(needs.setup.outputs.matrix) }} timeout-minutes: 90 - defaults: - run: - shell: bash -el {0} + steps: - - uses: actions/checkout@v4 + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 with: - fetch-depth: 0 - - uses: conda-incubator/setup-miniconda@v3 + environments: ${{ matrix.environment }} + - name: Test Unit + run: pixi run -e ${{ matrix.environment }} test-unit-cov + - name: Test Geo + run: pixi run -e ${{ matrix.environment }} test-unit-geo-cov + - name: Test Examples + run: pixi run -e ${{ matrix.environment }} test-example + - uses: codecov/codecov-action@v4 with: - auto-update-conda: true - environment-file: envs/py${{ matrix.python-version }}-tests.yaml - activate-environment: hvplottests - - name: conda info - run: conda info - - name: conda list - run: conda list - - name: bokeh sampledata - if: ${{ matrix.python-version == '3.9'}} - run: bokeh sampledata - - name: unit tests - run: pytest -v hvplot --cov=hvplot --cov-append - - name: unit tests geo - run: pytest -v hvplot --geo --cov=hvplot --cov-append - - name: examples tests - run: pytest -n logical --dist loadscope --nbval-lax -p no:python + token: ${{ secrets.CODECOV_TOKEN }} + + core_test_suite: + name: core:${{ matrix.environment }}:${{ matrix.os }} + needs: [pre_commit, setup, pixi_lock] + runs-on: ${{ matrix.os }} + if: needs.setup.outputs.code_change == 'true' + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest"] + environment: ["test-core"] + timeout-minutes: 120 + steps: + - uses: holoviz-dev/holoviz_tasks/pixi_install@v0 + with: + environments: ${{ matrix.environment }} + - name: Test Unit + run: pixi run -e ${{ matrix.environment }} test-unit + pip_test: name: pip tests:${{ matrix.os }}:${{ matrix.python-version }} needs: [pre_commit, setup] + if: needs.setup.outputs.matrix_option != 'default' timeout-minutes: 90 runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: ${{ fromJson(needs.setup.outputs.matrix) }} - defaults: - run: - shell: bash -e {0} steps: - uses: actions/checkout@v4 with: @@ -181,3 +199,13 @@ jobs: verbose: false env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + result_test_suite: + name: result:test + needs: [unit_test_suite, ui_test_suite, core_test_suite, pip_test] + if: always() + runs-on: ubuntu-latest + steps: + - name: check for failures + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: echo job failed && exit 1 diff --git a/.gitignore b/.gitignore index ba51ceabd..e6f221e08 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,7 @@ doc/user_guide/plot.html # setuptools_scm hvplot/_version.py + +# pixi +.pixi +pixi.lock diff --git a/pixi.toml b/pixi.toml new file mode 100644 index 000000000..c3bf8c973 --- /dev/null +++ b/pixi.toml @@ -0,0 +1,208 @@ +[project] +name = "hvplot" +channels = ["pyviz/label/dev", "conda-forge"] +platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] + +[tasks] +download-data = 'python scripts/download_data.py' +install = 'python -m pip install --no-deps --disable-pip-version-check -e .' + +[activation.env] +PYTHONIOENCODING = "utf-8" + +[environments] +test-39 = ["py39", "test-core", "test", "example", "geo", "graphviz", "test-example"] +test-310 = ["py310", "test-core", "test", "example", "geo", "graphviz", "test-example"] +test-311 = ["py311", "test-core", "test", "example", "geo", "graphviz", "test-example"] +test-312 = ["py312", "test-core", "test", "example", "geo", "graphviz", "test-example"] +# test-313 = ["py313", "test-core", "test", "example", "geo", "graphviz", "test-example"] +test-core = ["py313", "test-core"] +docs = ["py311", "doc", "example", "geo", "graphviz"] +build = ["py311", "build"] +lint = ["py311", "lint"] + +[dependencies] +nomkl = "*" +pip = "*" +# Required +bokeh = ">=3.1" +colorcet = ">=2" +holoviews = ">=1.19.0" +numpy = ">=1.21" +packaging = "*" +pandas = ">=1.3" +panel = ">=1.0" +param = ">=1.12.0,<3.0" + +[feature.py39.dependencies] +python = "3.9.*" + +[feature.py310.dependencies] +python = "3.10.*" +bokeh_sampledata = "*" + +[feature.py311.dependencies] +python = "3.11.*" +bokeh_sampledata = "*" + +[feature.py312.dependencies] +python = "3.12.*" +bokeh_sampledata = "*" + +[feature.py312.activation.env] +COVERAGE_CORE = "sysmon" + +[feature.py313.dependencies] +python = "3.13.*" +bokeh_sampledata = "*" + +[feature.py313.activation.env] +COVERAGE_CORE = "sysmon" + +# =================== SHARED DEPS =================== + +# Dependencies required to run the notebooks +[feature.example.dependencies] +dask-core = "*" # dask[dataframe] +dask-expr = "*" # dask[dataframe] +datashader = ">=0.6.5" +duckdb = "*" +# In 0.9 fugue added the sql extra but didn't add a fugue-sql conda package, +# removing the sql deps from core fugue. Adding them manually here. +qpd = ">=0.4.4" +fugue-sql-antlr = ">=0.2.0" +sqlglot = "*" +jinja2 = "*" +# fugue[sql] = "*" +fugue = "*" +# end fugue +"ibis-duckdb" = "*" # ibis-framework[duckdb] +intake-parquet = ">=0.2.3" +intake-xarray = ">=0.5.0,<2" +intake = ">=0.6.5,<2.0.0" +ipywidgets = "*" +networkx = ">=2.6.3" +matplotlib = "*" +notebook = ">=5.4" +numba = ">=0.51.0" +pillow = ">=8.2.0" +plotly = "*" +polars = "*" +pooch = ">=1.6.0" +s3fs = ">=2022.1.0" +scikit-image = ">=0.17.2" +scipy = ">=1.5.3" +selenium = ">=3.141.0" +streamz = ">=0.3.0" +xarray = ">=0.18.2" +xyzservices = ">=2022.9.0" +geodatasets = ">=2023.12.0" + +# =================== TESTS =================== + +[feature.test-core.dependencies] +# Minimum dependencies required to run the test suite. +dask-core = "*" # dask[dataframe] +dask-expr = "*" # dask[dataframe] +ipywidgets = "*" +matplotlib = "*" +parameterized = "*" +plotly = "*" +pooch = "*" +pre-commit = "*" +pytest-cov = "*" +pytest = "*" +ruff = "*" +scipy = "*" +xarray = "*" +# bokeh_sampledata = "*" (only available starting from Python 3.10) +psutil = "*" + +[feature.test-example.dependencies] +# Dependencies required to run the examples notebooks. +pytest-xdist = "*" +nbval = "*" + +[feature.test.dependencies] +# Optional tests dependencies, i.e. one should be able to run and pass +# the test suite without installing any of those. + +# In 0.9 fugue added the sql extra but didn't add a fugue-sql conda package, +# removing the sql deps from core fugue. Adding them manually here. +qpd = ">=0.4.4" +fugue-sql-antlr = ">=0.2.0" +sqlglot = "*" +jinja2 = "*" +# fugue[sql] = "*" +fugue = "*" +# end fugue +"ibis-duckdb" = "*" # ibis-framework[duckdb] +polars = "*" +dask = "*" +spatialpandas = "*" +duckdb = "*" + +[feature.geo.dependencies] +# Geo dependencies are always a little special. +cartopy = "*" +fiona = "*" +geopandas = "*" +geoviews-core = ">=1.9.0" # geoviews +pyproj = "*" +rasterio = "*" +rioxarray = "*" +spatialpandas = ">=0.4.3" + +[feature.graphviz.dependencies] +# graphviz is difficult to install with pip, ok with conda. +pygraphviz = "*" + +[feature.test.tasks] +test-unit = 'pytest -v hvplot' +test-unit-geo = 'pytest -v hvplot --geo' +test-unit-cov = 'pytest -v hvplot --cov=hvplot --cov-append' +test-unit-geo-cov = 'pytest -v hvplot --geo --cov=hvplot --cov-append' + +[feature.test-example.tasks] +test-example = 'pytest -n logical --dist loadscope --nbval-lax -p no:python' + +# =================== DOCS ==================== + +[feature.doc] +channels = [ + # To get dev HoloViews, not always needed. + "pyviz/label/dev", + # To get dev nbsite, not always needed. + "pyviz/label/tooling_dev", + "conda-forge" +] +dependencies = {nbsite = ">=0.8.6", sphinxext-rediraffe = "*"} + +[feature.doc.activation.env] +MOZ_HEADLESS = "1" +MPLBACKEND = "Agg" +DISPLAY = ":99.0" + +[feature.doc.tasks] +docs-build = 'sphinx-build -b html doc builtdocs' + +# ================== BUILD ==================== + +[feature.build.dependencies] +python-build = "*" +conda-build = "*" +anaconda-client = "*" + +[feature.build.tasks] +build-conda = 'bash scripts/conda/build.sh' +publish-conda = 'anaconda' +build-pip = 'python -m build .' + +# =================== LINT ==================== + +[feature.lint.dependencies] +pre-commit = "*" + +[feature.lint.tasks] +lint = 'pre-commit run --all-files' +lint-install = 'pre-commit install' diff --git a/scripts/conda/build.sh b/scripts/conda/build.sh new file mode 100644 index 000000000..bcbb20de6 --- /dev/null +++ b/scripts/conda/build.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +PACKAGE="hvplot" + +python -m build --sdist . + +VERSION=$(python -c "import $PACKAGE; print($PACKAGE._version.__version__)") +export VERSION + +conda build scripts/conda/recipe --no-anaconda-upload --no-verify -c conda-forge --package-format 1 + +mv "$CONDA_PREFIX/conda-bld/noarch/$PACKAGE-$VERSION-py_0.tar.bz2" dist diff --git a/conda.recipe/meta.yaml b/scripts/conda/recipe/meta.yaml similarity index 89% rename from conda.recipe/meta.yaml rename to scripts/conda/recipe/meta.yaml index f90b50a94..5bbee249a 100644 --- a/conda.recipe/meta.yaml +++ b/scripts/conda/recipe/meta.yaml @@ -1,4 +1,4 @@ -{% set pyproject = load_file_data('../pyproject.toml', from_recipe_dir=True) %} +{% set pyproject = load_file_data('../../../pyproject.toml', from_recipe_dir=True) %} {% set buildsystem = pyproject['build-system'] %} {% set project = pyproject['project'] %} @@ -10,7 +10,7 @@ package: version: {{ version }} source: - path: .. + url: ../../../dist/{{ name }}-{{ version }}.tar.gz build: noarch: python diff --git a/scripts/download_data.py b/scripts/download_data.py new file mode 100644 index 000000000..0a3e5279e --- /dev/null +++ b/scripts/download_data.py @@ -0,0 +1,17 @@ +import bokeh +from packaging.version import Version + +if Version(bokeh.__version__).release < (3, 5, 0): + import bokeh.sampledata + + bokeh.sampledata.download() + +try: + import pooch # noqa: F401 + import scipy # noqa: F401 + import xarray as xr + + xr.tutorial.open_dataset('air_temperature') + xr.tutorial.open_dataset('rasm') +except ModuleNotFoundError as e: + print(f'ModuleNotFoundError when attempting to download xarray datasets : {e}') From 6d535ec8d3f1d6fadd0dc799d13ed113a3e35e18 Mon Sep 17 00:00:00 2001 From: maximlt Date: Mon, 20 Jan 2025 20:26:51 +0100 Subject: [PATCH 02/14] remove pyproject2conda --- .pre-commit-config.yaml | 9 ---- envs/py3.10-tests.yaml | 80 ------------------------------------ envs/py3.11-docs.yaml | 70 ------------------------------- envs/py3.11-tests.yaml | 80 ------------------------------------ envs/py3.12-tests.yaml | 80 ------------------------------------ envs/py3.9-tests.yaml | 79 ----------------------------------- pyproject.toml | 25 ----------- scripts/update_conda_envs.py | 27 ------------ 8 files changed, 450 deletions(-) delete mode 100644 envs/py3.10-tests.yaml delete mode 100644 envs/py3.11-docs.yaml delete mode 100644 envs/py3.11-tests.yaml delete mode 100644 envs/py3.12-tests.yaml delete mode 100644 envs/py3.9-tests.yaml delete mode 100644 scripts/update_conda_envs.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 933b7741a..71ed4ffcf 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,12 +30,3 @@ repos: exclude: (\.min\.js$|\.svg$|\.html$) additional_dependencies: - tomli - - repo: local - hooks: - - id: update_conda_envs - name: Update conda envs - entry: python scripts/update_conda_envs.py - language: python - additional_dependencies: [pyproject2conda] - files: "(pyproject.toml|update_conda_envs)" - pass_filenames: false diff --git a/envs/py3.10-tests.yaml b/envs/py3.10-tests.yaml deleted file mode 100644 index ab421d415..000000000 --- a/envs/py3.10-tests.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# This file is autogenerated by pyproject2conda -# with the following command: -# -# $ pyproject2conda project --overwrite force --template-python envs/py{py_version}-{env} -# -# You should not manually edit this file. -# Instead edit the corresponding pyproject.toml file. -# -name: hvplottests -channels: - - nodefaults - - pyviz/label/dev - - conda-forge -dependencies: - - python=3.10 - - bokeh>=3.1 - - bokeh_sampledata - - cartopy - - colorcet>=2 - - dask - - dask>=2021.3.0 - - datashader>=0.6.5 - - duckdb - - fiona - - fugue - - fugue-sql-antlr>=0.2.0 - - geodatasets>=2023.12.0 - - geopandas - - geoviews-core>=1.9.0 - - holoviews>=1.19.0 - - ibis-duckdb - - intake-parquet>=0.2.3 - - intake-xarray<2,>=0.5.0 - - intake<2.0.0,>=0.6.5 - - ipywidgets - - jinja2 - - matplotlib - - nbval - - networkx>=2.6.3 - - notebook>=5.4 - - numba>=0.51.0 - - numpy>=1.21 - - packaging - - pandas>=1.3 - - panel>=1.0 - - param<3.0,>=1.12.0 - - parameterized - - pillow>=8.2.0 - - plotly - - polars - - pooch - - pooch>=1.6.0 - - pre-commit - - psutil - - pygraphviz - - pyproj - - pytest - - pytest-cov - - pytest-xdist - - qpd>=0.4.4 - - rasterio - - rioxarray - - ruff - - s3fs>=2022.1.0 - - scikit-image>=0.17.2 - - scipy - - scipy>=1.5.3 - - selenium>=3.141.0 - - setuptools_scm>=6 - - spatialpandas - - spatialpandas>=0.4.3 - - sqlglot - - streamz>=0.3.0 - - xarray - - xarray>=0.18.2 - - xyzservices>=2022.9.0 - - pip - - pip: - - -e .. diff --git a/envs/py3.11-docs.yaml b/envs/py3.11-docs.yaml deleted file mode 100644 index 432c0edb4..000000000 --- a/envs/py3.11-docs.yaml +++ /dev/null @@ -1,70 +0,0 @@ -# -# This file is autogenerated by pyproject2conda -# with the following command: -# -# $ pyproject2conda project --overwrite force --template-python envs/py{py_version}-{env} -# -# You should not manually edit this file. -# Instead edit the corresponding pyproject.toml file. -# -name: hvplotdocs -channels: - - nodefaults - - pyviz/label/dev - - pyviz/label/tooling_dev - - conda-forge -dependencies: - - python=3.11 - - bokeh>=3.1 - - bokeh_sampledata - - cartopy - - colorcet>=2 - - dask>=2021.3.0 - - datashader>=0.6.5 - - duckdb - - fiona - - fugue - - fugue-sql-antlr>=0.2.0 - - geodatasets>=2023.12.0 - - geopandas - - geoviews-core>=1.9.0 - - holoviews>=1.19.0 - - ibis-duckdb - - intake-parquet>=0.2.3 - - intake-xarray<2,>=0.5.0 - - intake<2.0.0,>=0.6.5 - - ipywidgets - - jinja2 - - matplotlib - - nbsite>=0.8.6 - - networkx>=2.6.3 - - notebook>=5.4 - - numba>=0.51.0 - - numpy>=1.21 - - packaging - - pandas>=1.3 - - panel>=1.0 - - param<3.0,>=1.12.0 - - pillow>=8.2.0 - - plotly - - polars - - pooch>=1.6.0 - - pygraphviz - - pyproj - - qpd>=0.4.4 - - rasterio - - rioxarray - - s3fs>=2022.1.0 - - scikit-image>=0.17.2 - - scipy>=1.5.3 - - selenium>=3.141.0 - - setuptools_scm>=6 - - spatialpandas>=0.4.3 - - sphinxext-rediraffe - - sqlglot - - streamz>=0.3.0 - - xarray>=0.18.2 - - xyzservices>=2022.9.0 - - pip - - pip: - - -e .. diff --git a/envs/py3.11-tests.yaml b/envs/py3.11-tests.yaml deleted file mode 100644 index 2ceb1523a..000000000 --- a/envs/py3.11-tests.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# This file is autogenerated by pyproject2conda -# with the following command: -# -# $ pyproject2conda project --overwrite force --template-python envs/py{py_version}-{env} -# -# You should not manually edit this file. -# Instead edit the corresponding pyproject.toml file. -# -name: hvplottests -channels: - - nodefaults - - pyviz/label/dev - - conda-forge -dependencies: - - python=3.11 - - bokeh>=3.1 - - bokeh_sampledata - - cartopy - - colorcet>=2 - - dask - - dask>=2021.3.0 - - datashader>=0.6.5 - - duckdb - - fiona - - fugue - - fugue-sql-antlr>=0.2.0 - - geodatasets>=2023.12.0 - - geopandas - - geoviews-core>=1.9.0 - - holoviews>=1.19.0 - - ibis-duckdb - - intake-parquet>=0.2.3 - - intake-xarray<2,>=0.5.0 - - intake<2.0.0,>=0.6.5 - - ipywidgets - - jinja2 - - matplotlib - - nbval - - networkx>=2.6.3 - - notebook>=5.4 - - numba>=0.51.0 - - numpy>=1.21 - - packaging - - pandas>=1.3 - - panel>=1.0 - - param<3.0,>=1.12.0 - - parameterized - - pillow>=8.2.0 - - plotly - - polars - - pooch - - pooch>=1.6.0 - - pre-commit - - psutil - - pygraphviz - - pyproj - - pytest - - pytest-cov - - pytest-xdist - - qpd>=0.4.4 - - rasterio - - rioxarray - - ruff - - s3fs>=2022.1.0 - - scikit-image>=0.17.2 - - scipy - - scipy>=1.5.3 - - selenium>=3.141.0 - - setuptools_scm>=6 - - spatialpandas - - spatialpandas>=0.4.3 - - sqlglot - - streamz>=0.3.0 - - xarray - - xarray>=0.18.2 - - xyzservices>=2022.9.0 - - pip - - pip: - - -e .. diff --git a/envs/py3.12-tests.yaml b/envs/py3.12-tests.yaml deleted file mode 100644 index 7f1478483..000000000 --- a/envs/py3.12-tests.yaml +++ /dev/null @@ -1,80 +0,0 @@ -# -# This file is autogenerated by pyproject2conda -# with the following command: -# -# $ pyproject2conda project --overwrite force --template-python envs/py{py_version}-{env} -# -# You should not manually edit this file. -# Instead edit the corresponding pyproject.toml file. -# -name: hvplottests -channels: - - nodefaults - - pyviz/label/dev - - conda-forge -dependencies: - - python=3.12 - - bokeh>=3.1 - - bokeh_sampledata - - cartopy - - colorcet>=2 - - dask - - dask>=2021.3.0 - - datashader>=0.6.5 - - duckdb - - fiona - - fugue - - fugue-sql-antlr>=0.2.0 - - geodatasets>=2023.12.0 - - geopandas - - geoviews-core>=1.9.0 - - holoviews>=1.19.0 - - ibis-duckdb - - intake-parquet>=0.2.3 - - intake-xarray<2,>=0.5.0 - - intake<2.0.0,>=0.6.5 - - ipywidgets - - jinja2 - - matplotlib - - nbval - - networkx>=2.6.3 - - notebook>=5.4 - - numba>=0.51.0 - - numpy>=1.21 - - packaging - - pandas>=1.3 - - panel>=1.0 - - param<3.0,>=1.12.0 - - parameterized - - pillow>=8.2.0 - - plotly - - polars - - pooch - - pooch>=1.6.0 - - pre-commit - - psutil - - pygraphviz - - pyproj - - pytest - - pytest-cov - - pytest-xdist - - qpd>=0.4.4 - - rasterio - - rioxarray - - ruff - - s3fs>=2022.1.0 - - scikit-image>=0.17.2 - - scipy - - scipy>=1.5.3 - - selenium>=3.141.0 - - setuptools_scm>=6 - - spatialpandas - - spatialpandas>=0.4.3 - - sqlglot - - streamz>=0.3.0 - - xarray - - xarray>=0.18.2 - - xyzservices>=2022.9.0 - - pip - - pip: - - -e .. diff --git a/envs/py3.9-tests.yaml b/envs/py3.9-tests.yaml deleted file mode 100644 index 0d002c0cf..000000000 --- a/envs/py3.9-tests.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# -# This file is autogenerated by pyproject2conda -# with the following command: -# -# $ pyproject2conda project --overwrite force --template-python envs/py{py_version}-{env} -# -# You should not manually edit this file. -# Instead edit the corresponding pyproject.toml file. -# -name: hvplottests -channels: - - nodefaults - - pyviz/label/dev - - conda-forge -dependencies: - - python=3.9 - - bokeh>=3.1 - - cartopy - - colorcet>=2 - - dask - - dask>=2021.3.0 - - datashader>=0.6.5 - - duckdb - - fiona - - fugue - - fugue-sql-antlr>=0.2.0 - - geodatasets>=2023.12.0 - - geopandas - - geoviews-core>=1.9.0 - - holoviews>=1.19.0 - - ibis-duckdb - - intake-parquet>=0.2.3 - - intake-xarray<2,>=0.5.0 - - intake<2.0.0,>=0.6.5 - - ipywidgets - - jinja2 - - matplotlib - - nbval - - networkx>=2.6.3 - - notebook>=5.4 - - numba>=0.51.0 - - numpy>=1.21 - - packaging - - pandas>=1.3 - - panel>=1.0 - - param<3.0,>=1.12.0 - - parameterized - - pillow>=8.2.0 - - plotly - - polars - - pooch - - pooch>=1.6.0 - - pre-commit - - psutil - - pygraphviz - - pyproj - - pytest - - pytest-cov - - pytest-xdist - - qpd>=0.4.4 - - rasterio - - rioxarray - - ruff - - s3fs>=2022.1.0 - - scikit-image>=0.17.2 - - scipy - - scipy>=1.5.3 - - selenium>=3.141.0 - - setuptools_scm>=6 - - spatialpandas - - spatialpandas>=0.4.3 - - sqlglot - - streamz>=0.3.0 - - xarray - - xarray>=0.18.2 - - xyzservices>=2022.9.0 - - pip - - pip: - - -e .. diff --git a/pyproject.toml b/pyproject.toml index 435faafcd..ba228da39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -210,28 +210,3 @@ extend-select = [ [tool.ruff.format] quote-style = "single" - -[tool.pyproject2conda.dependencies] -geoviews = { skip = true, packages = "geoviews-core >=1.9.0" } -# It should be ibis-framework[duckdb], but it works anyway. -ibis-framework = { skip = true, packages = "ibis-duckdb" } - -[tool.pyproject2conda.envs."tests"] -channels = ["nodefaults", "pyviz/label/dev", "conda-forge"] -python = ["3.9", "3.10", "3.11", "3.12"] -extras = ["tests", "examples-tests", "geo", "graphviz", "dev-extras"] -name = "hvplottests" -# reqs = ["-e .."] # Doesn't work - -[tool.pyproject2conda.envs."docs"] -channels = [ - "nodefaults", - # To get dev HoloViews, not always needed. - "pyviz/label/dev", - # To get dev nbsite, not always needed. - "pyviz/label/tooling_dev", - "conda-forge" -] -python = ["3.11"] -extras = ["doc", "examples", "geo", "graphviz", "dev-extras"] -name = "hvplotdocs" diff --git a/scripts/update_conda_envs.py b/scripts/update_conda_envs.py deleted file mode 100644 index 13c8730a6..000000000 --- a/scripts/update_conda_envs.py +++ /dev/null @@ -1,27 +0,0 @@ -import subprocess -from pathlib import Path - - -def main(): - subprocess.run( - [ - 'pyproject2conda', - 'project', - '--overwrite', - 'force', - '--template-python', - str(Path('envs', 'py{py_version}-{env}')), - ], - check=True, - ) - - # Hacky way to install the package in editable mode when creating the env. - for file in Path('envs').glob('*.yaml'): - with open(file, 'a', encoding='utf-8') as f: - f.write(' - pip\n') - f.write(' - pip:\n') - f.write(' - -e ..\n') - - -if __name__ == '__main__': - main() From 54a0d8ad58d1d192fd43559822d4b4db5eea3e49 Mon Sep 17 00:00:00 2001 From: maximlt Date: Mon, 20 Jan 2025 20:34:06 +0100 Subject: [PATCH 03/14] fix job needs --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f695d1585..5d4e92a9c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -202,7 +202,7 @@ jobs: result_test_suite: name: result:test - needs: [unit_test_suite, ui_test_suite, core_test_suite, pip_test] + needs: [unit_test_suite, core_test_suite, pip_test] if: always() runs-on: ubuntu-latest steps: From a2f34cdcd3991d344e0878a54474ec3ba58ef20a Mon Sep 17 00:00:00 2001 From: maximlt Date: Mon, 20 Jan 2025 20:43:13 +0100 Subject: [PATCH 04/14] replace python-version with environment --- .github/workflows/test.yaml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5d4e92a9c..b390ad4d2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -65,10 +65,10 @@ jobs: run: | MATRIX=$(jq -nsc '{ "os": ["ubuntu-latest", "macos-latest", "windows-latest"], - "python-version": ["3.9", "3.12"], + "environment": ["test-39", "test-312"], "exclude": [ { - "python-version": "3.9", + "environment": "test-39", "os": "macos-latest" } ] @@ -79,24 +79,24 @@ jobs: run: | MATRIX=$(jq -nsc '{ "os": ["ubuntu-latest", "macos-latest", "windows-latest"], - "python-version": ["3.9", "3.12"], + "environment": ["test-39", "test-312"], "include": [ { - "python-version": "3.9", + "environment": "test-39", "os": "ubuntu-latest" }, { - "python-version": "3.10", + "environment": "test-310", "os": "ubuntu-latest" }, { - "python-version": "3.11", + "environment": "test-311", "os": "ubuntu-latest" } ], "exclude": [ { - "python-version": "3.9", + "environment": "test-39", "os": "macos-latest" } ] @@ -107,7 +107,7 @@ jobs: run: | MATRIX=$(jq -nsc '{ "os": ["ubuntu-latest"], - "python-version": ["3.12"] + "environment": ["test-312"] }') echo "MATRIX=$MATRIX" >> $GITHUB_ENV @@ -161,7 +161,7 @@ jobs: run: pixi run -e ${{ matrix.environment }} test-unit pip_test: - name: pip tests:${{ matrix.os }}:${{ matrix.python-version }} + name: pip tests:${{ matrix.os }}:${{ matrix.environment }} needs: [pre_commit, setup] if: needs.setup.outputs.matrix_option != 'default' timeout-minutes: 90 @@ -175,7 +175,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: '3.12' - name: install with geo run: python -m pip install -v --prefer-binary -e '.[tests, examples-tests, geo, hvdev, hvdev-geo, dev-extras]' - name: python version and pip list @@ -183,7 +183,6 @@ jobs: python --version --version python -m pip list - name: bokeh sampledata - if: ${{ matrix.python-version == '3.9'}} run: bokeh sampledata - name: unit tests run: pytest -v hvplot --cov=hvplot --cov-append From baf990e6390fff1b7be6aa08801964745e61825e Mon Sep 17 00:00:00 2001 From: maximlt Date: Mon, 20 Jan 2025 20:44:45 +0100 Subject: [PATCH 05/14] enable core suite --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b390ad4d2..c82bfd830 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -146,7 +146,6 @@ jobs: name: core:${{ matrix.environment }}:${{ matrix.os }} needs: [pre_commit, setup, pixi_lock] runs-on: ${{ matrix.os }} - if: needs.setup.outputs.code_change == 'true' strategy: fail-fast: false matrix: From 2ba06e2ee8b74222b43f4c11bec913beefbebddd Mon Sep 17 00:00:00 2001 From: maximlt Date: Tue, 21 Jan 2025 21:56:30 +0100 Subject: [PATCH 06/14] simplify dev flow and add docs --- doc/conf.py | 4 +- doc/developer_guide.md | 240 +++++++++++++++++++++++++++++++++++ doc/developer_guide/index.md | 176 ------------------------- doc/index.md | 2 +- pixi.toml | 85 ++++++++++--- scripts/download_data.py | 2 + scripts/sync_git_tags.py | 31 +++++ 7 files changed, 345 insertions(+), 195 deletions(-) create mode 100644 doc/developer_guide.md delete mode 100644 doc/developer_guide/index.md create mode 100644 scripts/sync_git_tags.py diff --git a/doc/conf.py b/doc/conf.py index 2a5f16a85..b1e5d6993 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -98,7 +98,9 @@ rediraffe_redirects = { # Removal of the developer testing page - 'developer_guide/testing': 'developer_guide/index', + 'developer_guide/testing': 'developer_guide', + # Removal of the developer_guide folder + 'developer_guide/index': 'developer_guide', } html_context.update( # noqa diff --git a/doc/developer_guide.md b/doc/developer_guide.md new file mode 100644 index 000000000..653cbd892 --- /dev/null +++ b/doc/developer_guide.md @@ -0,0 +1,240 @@ +(devguide-setup)= + +# Developer Guide + +The hvPlot library is a project that provides a wide range of data interfaces and an extensible set of plotting backends, which means the development and testing process involves a broad set of libraries. + +This guide describes how to install and configure development environments. + +If you have any problems with the steps here, please reach out in the `dev` channel on [Discord](https://discord.gg/rb6gPXbdAr) or on [Discourse](https://discourse.holoviz.org/). + +## TL;DR + +0. Open an [issue on Github](https://github.com/holoviz/hvplot/issues) if needed +1. Fork and clone [hvPlot's Github repository](https://github.com/holoviz/hvplot) +2. Install [`pixi`](https://pixi.sh) +3. Run `pixi run install-dev` to create your development environment +4. Make some changes and run: + - `pixi run test-unit` if you updated the source code to run the unit tests + - `pixi run test-example` if you updated the notebooks to run them + - `pixi run docs-build-dev` if you need to build the website locally +5. Open a Pull Request + +## Preliminaries + +### Basic understanding of how to contribute to Open Source + +If this is your first open-source contribution, please study one +or more of the below resources. + +- [How to Get Started with Contributing to Open Source | Video](https://youtu.be/RGd5cOXpCQw) +- [Contributing to Open-Source Projects as a New Python Developer | Video](https://youtu.be/jTTf4oLkvaM) +- [How to Contribute to an Open Source Python Project | Blog post](https://www.educative.io/blog/contribue-open-source-python-project) + +### Git + +The hvPlot source code is stored in a [Git](https://git-scm.com) source control repository. The first step to working on hvPlot is to install Git onto your system. There are different ways to do this, depending on whether you use Windows, Mac, or Linux. + +To install Git on any platform, refer to the [Installing Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) section of the [Pro Git Book](https://git-scm.com/book/en/v2). + +To contribute to hvPlot, you will also need [Github account](https://github.com/join) and knowledge of the [_fork and pull request workflow_](https://docs.github.com/en/get-started/quickstart/contributing-to-projects). + +### Pixi + +Developing all aspects of hvPlot requires a wide range of packages in different environments, but for new contributors the `default` environment will be more than enough. + +To make this more manageable, Pixi manages the developer experience. To install Pixi, follow [this guide](https://pixi.sh/latest/#installation). + +#### Glossary + +- *Tasks*: A task is what can be run with `pixi run `. Tasks can be anything from installing packages to running tests. +- *Environments*: An environment is a set of packages installed in a virtual environment. Each environment has a name; you can run tasks in a specific environment with the `-e` flag. For example, `pixi run -e test-core test-unit` will run the `test-unit` task in the `test-core` environment. +- *Lock-file*: A lock-file is a file that contains all the information about the environments. + +For more information, see the [Pixi documentation](https://pixi.sh/latest/). + +:::{admonition} Note +:class: info + +The first time you run `pixi`, it will create a `.pixi` directory in the source directory. +This directory will contain all the files needed for the virtual environments. +The `.pixi` directory can be large, so it is advised not to put the source directory into a cloud-synced directory. + +::: + +## Installing the Project + +### Cloning the Project + +The source code for the hvPlot project is hosted on [GitHub](https://github.com/holoviz/hvplot). The first thing you need to do is clone the repository. + +1. Go to [github.com/holoviz/hvplot](https://github.com/holoviz/hvplot) +2. [Fork the repository](https://docs.github.com/en/get-started/quickstart/contributing-to-projects#forking-a-repository) +3. Run in your terminal: `git clone https://github.com//hvplot` + +The instructions for cloning above created a `hvplot` directory at your file system location. +This `hvplot` directory is the _source checkout_ for the remainder of this document, and your current working directory is this directory. + +## Start developing + +To start developing, run the following command, this will create an environment called `default` (in `.pixi/envs`), install hvPlot in [editable mode](https://pip.pypa.io/en/stable/topics/local-project-installs/#editable-installs), download test datasets, and install `pre-commit`: + +```bash +pixi run install-dev +``` + +:::{admonition} Note +:class: info + +The first time you run it, it will create a `pixi.lock` file with information for all available environments. +This command will take a minute or so to run. +::: + +All available tasks can be found by running `pixi task list`, the following sections will give a brief introduction to the most common tasks. + +### Syncing Git tags with upstream repository + +If you are working from a forked repository of hvPlot, you will need to sync the tags with the upstream repository. +This is needed because the hvPlot version number depends on [`git tags`](https://git-scm.com/book/en/v2/Git-Basics-Tagging). +Syncing the git tags can be done with: + +```bash +pixi run sync-git-tags +``` + +## Developer Environment + +The `default` environment is meant to provide all the tools needed to develop hvPlot. + +This environment is created by running `pixi run install-dev`. Run `pixi shell` to activate it; this is equivalent to `source venv/bin/activate` in a Python virtual environment or `conda activate` in a conda environment. + +If you need to run a command directly instead of via `pixi`, activate the environment and run the command (e.g. `pixi shell` and `pytest hvplot/tests/`). + +### VS Code + +This environment can also be selected in your IDE. In VS Code, this can be done by running the command `Python: Select Interpreter` and choosing `{'default': Pixi}`. + +

+ 001 + 002 +

+ +To confirm you are using this dev environment, check the bottom right corner: + +![003](https://assets.holoviews.org/static/dev_guide/003.png) + +### Jupyter Lab + +You can launch Jupyter lab with the `default` environment with `pixi run lab`. This can be advantageous when you need to edit the documentation or debug an example notebook. + +## Linting + +hvPlot uses [`pre-commit`](https://pre-commit.com/) to lint and format the source code. `pre-commit` is installed automatically when running `pixi run install-dev`; it can also be installed with `pixi run lint-install`. +`pre-commit` runs all the linters when a commit is made locally. Linting can be forced to run for all the files with: + +```bash +pixi run lint +``` + +:::{admonition} Note +:class: info + +Alternatively, if you have `pre-commit` installed elsewhere you can run: + +```bash +pre-commit install # To install +pre-commit run --all-files # To run on all files +``` + +::: + +## Testing + +To help keep hvPlot maintainable, all Pull Requests (PR) with code changes should typically be accompanied by relevant tests. While exceptions may be made for specific circumstances, the default assumption should be that a Pull Request without tests will not be merged. + +There are three types of tasks and five environments related to tests. + +### Unit tests + +Unit tests are usually small tests executed with [pytest](https://docs.pytest.org). They can be found in `hvplot/tests/`. +Unit tests can be run with the `test-unit` task: + +```bash +pixi run test-unit +``` + +:::{admonition} Advanced usage +:class: tip + +The task is available in the following environments: `test-39`, `test-310`, `test-311`, `test-312`, and `test-core`. Where the first ones have the same environments except for different Python versions, and `test-core` only has a core set of dependencies. + +You can run the task in a specific environment with the `-e` flag. For example, to run the `test-unit` task in the `test-39` environment, you can run: + +```bash +pixi run -e test-39 test-unit +``` + +::: + +:::{admonition} Advanced usage +:class: tip + +Currently, an editable install needs to be run in each environment. So, if you want to install in the `test-core` environment, you can add `--environment` / `-e` to the command: + +```bash +pixi run -e test-core install +``` + +::: + +### Example tests + +hvPlot's documentation consists mainly of Jupyter Notebooks. The example tests execute all the notebooks and fail if an error is raised. Example tests are possible thanks to [nbval](https://nbval.readthedocs.io/) and can be found in the `doc/` folder. +Example tests can be run with the following command: + +```bash +pixi run test-example +``` + +## Documentation + +The documentation can be built with the command: + +```bash +pixi run docs-build-dev +``` + +As hvPlot uses notebooks for much of the documentation, this takes a little while. You can disable building the gallery by setting the environment variable `HVPLOT_REFERENCE_GALLERY` to `false`. + +A development version of hvPlot can be found [here](https://holoviz-dev.github.io/hvplot/). You can ask a maintainer if they want to make a dev release for your PR, but there is no guarantee they will say yes. + +## Build + +hvPlot has two build tasks to build a Python (for pypi.org) and a Conda package (for anaconda.org). + +```bash +pixi run build-pip +pixi run build-conda +``` + +## Continuous Integration + +Every push to the `main` branch or any PR branch on GitHub automatically triggers a test build with [GitHub Actions](https://github.com/features/actions). + +You can see the list of all current and previous builds at [this URL](https://github.com/holoviz/hvplot/actions) + +### Etiquette + +GitHub Actions provides free build workers for open-source projects. A few considerations will help you be considerate of others needing these limited resources: + +- Run the tests locally before opening or pushing to an opened PR. + +- Group commits to meaningful chunks of work before pushing to GitHub (i.e., don't push on every commit). diff --git a/doc/developer_guide/index.md b/doc/developer_guide/index.md deleted file mode 100644 index 92808eea8..000000000 --- a/doc/developer_guide/index.md +++ /dev/null @@ -1,176 +0,0 @@ -(devguide-setup)= - -# Developer Guide - -```{contents} -:depth: 3 -:local: true -``` - -## Set up - -The hvPlot library is a complex project which provides a wide range -of data interfaces and an extensible set of plotting backends, which -means the development and testing process involves a wide set of -libraries. - -If you have any problems with the steps here, please contact the developers on [Discord](https://discord.gg/AXRHnJU6sP). - -### Preliminaries - -#### Git - -The hvPlot source code is stored in a [Git](https://git-scm.com) source control repository. -The first step to working on hvPlot is to install Git on to your system. -There are different ways to do this depending on whether, you are using -Windows, OSX, or Linux. - -To install Git on any platform, refer to the [Installing Git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) section of -the [Pro Git Book](https://git-scm.com/book/en/v2). - -#### Conda (optional) - -Developing hvPlot requires a wide range of dependencies that can all be installed with -the [conda package manager](https://conda.io). Using `conda` is sometimes the easiest way to install -a dependency (e.g. `graphviz`, Firefox drivers). However, these days most of the dependencies -required to develop hvPlot can be installed with `pip`. - -Follow [these instructions](https://conda.io/projects/conda/user-guide/install/index.html) to download conda. - -### Cloning the Repository - -The source code for the hvPlot project is hosted on GitHub. To clone the -source repository, issue the following command: - -```sh -git clone https://github.com/holoviz/hvplot.git -``` - -This will create a `hvplot` directory at your file system -location. This `hvplot` directory is referred to as the *source -checkout* for the remainder of this document. - -(dev-guide-installing-dependencies)= - -### Installing Dependencies - -hvPlot requires many additional packages for development and -testing. - -::::{tab-set} - -:::{tab-item} pip - -Start by creating a virtual environment with `venv`: - -``` -python -m venv .venv -``` - -Activate it: - -``` -# Linux/MacOs -source .venv/bin/activate -# Windows -.venv\Scripts\activate -``` - -Install the test dependencies: - -``` bash -pip install --prefer-binary -e '.[tests, examples-tests, geo, hvdev, hvdev-geo, dev-extras]' -``` - -::: - -:::{tab-item} conda - -Create a development conda environment using one of the environment files present -in the `./envs` folder, and activate it: - -``` bash -conda env create --file envs/py3.10-tests.yaml -conda activate hvplottests -``` - -::: - -:::: - - -### Setting up pre-commit - -hvPlot uses `pre-commit` to automatically apply linting to hvPlot code. -If you intend to contribute to hvPlot we recommend you enable it with: - -```sh -pre-commit install -``` - -This will ensure that every time you make a commit linting will automatically be applied. - - -## Testing - -This chapter describes how to run various tests locally in a -development environment, guidelines for writing tests, and information -regarding the continuous testing infrastructure. - -### Running Tests Locally - -Before attempting to run hvPlot tests, make sure you have successfully -run through all of the instructions in the {ref}`devguide-setup` -section of the Developer's Guide. - -Currently hvPlot uses linting two types of tests: regular unit tests -which are run with `pytest` and notebook example tests run with `pytest` and `nbval`: - -Run the unit tests with: - -```bash -pytest hvplot -pytest -v hvplot --geo # include the test that require geo dependencies -``` - -Run the example tests with: - -```sh -pytest -n auto --dist loadscope --nbval-lax -p no:python -``` - -### Writing Tests - -In order to help keep hvPlot maintainable, all Pull Requests that touch -code should normally be accompanied by relevant tests. While -exceptions may be made for specific circumstances, the default -assumption should be that a Pull Request without tests may not be -merged. - -Python unit tests maintain the basic functionality of the Python -portion of the hvPlot library. A few general guidelines will help you -write Python unit tests: - -In order to ensure that hvPlot's unit tests as relocatable and unambiguous -as possible, always prefer absolute imports in test files. When convenient, -import and use the entire module under test: - -- **Good**: `import hvplot.pandas` -- **Good**: `from hvplot.plotting import HvPlotTabular` -- **Bad**: `from ..plotting import HvPlotTabular` - -### Continuous Integration (CI) - -Every push to the `main` branch or any Pull Request branch on GitHub -automatically triggers a full test build on the [Github Action](https://github.com/holoviz/hvplot/actions) continuous -integration service. This is most often useful for running the full hvPlot -test suite continuously, but also triggers automated scripts for publishing -releases when a tagged branch is pushed. - -When in doubt about what command to run, you can always inspect the Github -workflow files in the `./github/workflows` folder so see what commands -are running on the CI. - -Github Action provides a limited number free build workers to Open Source projects. -Please be considerate of others and group commits into meaningful chunks of -work before pushing to GitHub (i.e. don't push on every commit). diff --git a/doc/index.md b/doc/index.md index 1f9396819..2b5bb7206 100644 --- a/doc/index.md +++ b/doc/index.md @@ -425,7 +425,7 @@ Getting Started User Guide Reference Gallery Topics -Developer Guide +Developer Guide Releases Roadmap About diff --git a/pixi.toml b/pixi.toml index dcd3aa00f..2c315f343 100644 --- a/pixi.toml +++ b/pixi.toml @@ -3,25 +3,53 @@ name = "hvplot" channels = ["pyviz/label/dev", "conda-forge"] platforms = ["linux-64", "osx-arm64", "osx-64", "win-64"] -[tasks] -download-data = 'python scripts/download_data.py' -install = 'python -m pip install --no-deps --disable-pip-version-check -e .' +[environments] +default = [ + "py312", + "required", + "test-core", + "test", + "example", + "geo", + "graphviz", + "test-example", + "lint", + "dev", +] -[activation.env] -PYTHONIOENCODING = "utf-8" +[environments.test-39] +features = ["py39", "required", ] +no-default-feature = true -[environments] -test-39 = ["py39", "test-core", "test", "example", "geo", "graphviz", "test-example"] -test-310 = ["py310", "test-core", "test", "example", "geo", "graphviz", "test-example"] -test-311 = ["py311", "test-core", "test", "example", "geo", "graphviz", "test-example"] -test-312 = ["py312", "test-core", "test", "example", "geo", "graphviz", "test-example"] -# test-313 = ["py313", "test-core", "test", "example", "geo", "graphviz", "test-example"] -test-core = ["py313", "test-core"] -docs = ["py311", "doc", "example", "geo", "graphviz"] -build = ["py311", "build"] -lint = ["py311", "lint"] - -[dependencies] +[environments.test-310] +features = ["py310", "required", "test-core", "test", "example", "geo", "graphviz", "test-example"] +no-default-feature = true + +[environments.test-311] +features = ["py311", "required", "test-core", "test", "example", "geo", "graphviz", "test-example"] +no-default-feature = true + +[environments.test-312] +features = ["py312", "required", "test-core", "test", "example", "geo", "graphviz", "test-example"] +no-default-feature = true + +[environments.test-core] +features = ["py313", "required", "test-core"] +no-default-feature = true + +[environments.docs] +features = ["py311", "required", "doc", "example", "geo", "graphviz"] +no-default-feature = true + +[environments.build] +features = ["py311", "required", "build"] +no-default-feature = true + +[environments.lint] +features = ["lint"] +no-default-feature = true + +[feature.required.dependencies] nomkl = "*" pip = "*" # Required @@ -34,6 +62,15 @@ pandas = ">=1.3" panel = ">=1.0" param = ">=1.12.0,<3.0" +[feature.required.tasks] +download-data = 'python scripts/download_data.py' +install = 'python -m pip install --no-deps --disable-pip-version-check -e .' +install-dev = { cmd = "pre-commit install", depends-on = ["install", "download-data"] } +sync-git-tags = 'python scripts/sync_git_tags.py hvplot' + +[feature.required.activation.env] +PYTHONIOENCODING = "utf-8" + [feature.py39.dependencies] python = "3.9.*" @@ -59,6 +96,16 @@ bokeh_sampledata = "*" [feature.py313.activation.env] COVERAGE_CORE = "sysmon" +# =================== DEV =================== + +[feature.dev.dependencies] +jupyterlab = "*" +jupyterlab-myst = "*" +setuptools_scm = ">=6" + +[feature.dev.tasks] +lab = 'jupyter lab' + # =================== SHARED DEPS =================== # Dependencies required to run the notebooks @@ -183,6 +230,10 @@ DISPLAY = ":99.0" [feature.doc.tasks] docs-build = 'sphinx-build -b html doc builtdocs' +# docs-build-dev = { cmd = "pre-commit install", depends-on = ["install", "docs-build"] } +_docs-install = 'python -m pip install --no-deps --disable-pip-version-check -e .' +docs-build-dev = { depends-on = ["_docs-install", "docs-build"] } + # ================== BUILD ==================== diff --git a/scripts/download_data.py b/scripts/download_data.py index 0a3e5279e..dbbaff4ed 100644 --- a/scripts/download_data.py +++ b/scripts/download_data.py @@ -5,6 +5,7 @@ import bokeh.sampledata bokeh.sampledata.download() + print('bokeh data downloaded.') try: import pooch # noqa: F401 @@ -13,5 +14,6 @@ xr.tutorial.open_dataset('air_temperature') xr.tutorial.open_dataset('rasm') + print('xarray data downloaded.') except ModuleNotFoundError as e: print(f'ModuleNotFoundError when attempting to download xarray datasets : {e}') diff --git a/scripts/sync_git_tags.py b/scripts/sync_git_tags.py new file mode 100644 index 000000000..9d217860f --- /dev/null +++ b/scripts/sync_git_tags.py @@ -0,0 +1,31 @@ +""" +Script to sync tags from upstream repository to forked repository +""" + +import sys +from subprocess import run + + +def main(package: str) -> None: + origin = run(['git', 'remote', 'get-url', 'origin'], check=True, capture_output=True) + upstream = run(['git', 'remote', 'get-url', 'upstream'], check=False, capture_output=True) + url = ( + f'https://github.com/holoviz/{package}.git' + if origin.stdout.startswith(b'http') + else f'git@github.com:holoviz/{package}.git' + ) + + if url == origin.stdout.strip().decode(): + print('Not a forked repository, exiting.') + return + elif upstream.returncode: + print(f'Adding {url!r} as remote upstream') + run(['git', 'remote', 'add', 'upstream', url], check=True, capture_output=True) + + print(f'Syncing tags from {package} repository with your forked repository') + run(['git', 'fetch', '--tags', 'upstream'], check=True, capture_output=True) + run(['git', 'push', '--tags'], check=True, capture_output=True) + + +if __name__ == '__main__': + main(sys.argv[1]) From 183f8f293a43e1d8f1996f4b746d7250c395549f Mon Sep 17 00:00:00 2001 From: maximlt Date: Tue, 21 Jan 2025 21:58:26 +0100 Subject: [PATCH 07/14] debug build --- .github/workflows/build.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e84d791ed..ed38b5d73 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -43,10 +43,12 @@ jobs: echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV - name: conda dev upload if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') - run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE + # run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE + run: echo "dev" - name: conda main upload if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) - run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE + # run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE + run: echo "prod" pip_build: name: Build PyPI @@ -64,4 +66,5 @@ jobs: run: pixi run -e build build-pip - name: Publish to PyPI if: github.event_name == 'push' - uses: pypa/gh-action-pypi-publish@release/v1 + # uses: pypa/gh-action-pypi-publish@release/v1 + run: echo "publish" From d1292184a61599d404729f81a1abe543cd37d07b Mon Sep 17 00:00:00 2001 From: maximlt Date: Tue, 21 Jan 2025 22:17:59 +0100 Subject: [PATCH 08/14] fix conda's build --- scripts/conda/recipe/meta.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/conda/recipe/meta.yaml b/scripts/conda/recipe/meta.yaml index 5bbee249a..271976918 100644 --- a/scripts/conda/recipe/meta.yaml +++ b/scripts/conda/recipe/meta.yaml @@ -33,8 +33,10 @@ test: requires: - pip {% for dep in project['optional-dependencies']['tests-core'] %} - {% if dep == 'dask[dataframe]' %} - - dask + {% if 'dask[dataframe]' in dep %} + - {{ dep | replace("[dataframe]", "") }} + {% elif 'bokeh_sampledata' in dep %} + - {{ dep | replace("; python_version >= '3.10'", "") }} {% else %} - {{ dep }} {% endif %} From 580ba89fd858e5e10fe158a70762efca11e11768 Mon Sep 17 00:00:00 2001 From: maximlt Date: Tue, 21 Jan 2025 22:37:56 +0100 Subject: [PATCH 09/14] fix the tests --- .github/workflows/test.yaml | 4 ++-- pixi.toml | 7 +++++-- pyproject.toml | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c82bfd830..a828ef18d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -181,8 +181,8 @@ jobs: run: | python --version --version python -m pip list - - name: bokeh sampledata - run: bokeh sampledata + - name: download datasets + run: python scripts/download_data.py - name: unit tests run: pytest -v hvplot --cov=hvplot --cov-append - name: unit tests geo diff --git a/pixi.toml b/pixi.toml index 2c315f343..1bac8c91c 100644 --- a/pixi.toml +++ b/pixi.toml @@ -18,7 +18,7 @@ default = [ ] [environments.test-39] -features = ["py39", "required", ] +features = ["py39", "required", "test-core", "test", "example", "geo", "graphviz", "test-example"] no-default-feature = true [environments.test-310] @@ -162,6 +162,7 @@ scipy = "*" xarray = "*" # bokeh_sampledata = "*" (only available starting from Python 3.10) psutil = "*" +cftime = "*" [feature.test-example.dependencies] # Dependencies required to run the examples notebooks. @@ -203,11 +204,13 @@ spatialpandas = ">=0.4.3" pygraphviz = "*" [feature.test.tasks] -test-unit = 'pytest -v hvplot' test-unit-geo = 'pytest -v hvplot --geo' test-unit-cov = 'pytest -v hvplot --cov=hvplot --cov-append' test-unit-geo-cov = 'pytest -v hvplot --geo --cov=hvplot --cov-append' +[feature.test-core.tasks] +test-unit = 'pytest -v hvplot' + [feature.test-example.tasks] test-example = 'pytest -n logical --dist loadscope --nbval-lax -p no:python' diff --git a/pyproject.toml b/pyproject.toml index df82750a9..528d55b29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ tests-core = [ "xarray", "bokeh_sampledata; python_version >= '3.10'", "psutil", + "cftime", ] # Optional tests dependencies, i.e. one should be able # to run and pass the test suite without installing any From 8ad57dcc0cead5cbff63928f3b711251320553d5 Mon Sep 17 00:00:00 2001 From: maximlt Date: Tue, 21 Jan 2025 22:57:14 +0100 Subject: [PATCH 10/14] undebug build --- .github/workflows/build.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ed38b5d73..e84d791ed 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -43,12 +43,10 @@ jobs: echo "CONDA_FILE=$(ls dist/*.tar.bz2)" >> $GITHUB_ENV - name: conda dev upload if: contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc') - # run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE - run: echo "dev" + run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev $CONDA_FILE - name: conda main upload if: (!(contains(env.TAG, 'a') || contains(env.TAG, 'b') || contains(env.TAG, 'rc'))) - # run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE - run: echo "prod" + run: pixi run -e build publish-conda --token ${{ secrets.CONDA_UPLOAD_TOKEN }} upload --user pyviz --label=dev --label=main $CONDA_FILE pip_build: name: Build PyPI @@ -66,5 +64,4 @@ jobs: run: pixi run -e build build-pip - name: Publish to PyPI if: github.event_name == 'push' - # uses: pypa/gh-action-pypi-publish@release/v1 - run: echo "publish" + uses: pypa/gh-action-pypi-publish@release/v1 From ab22ad0b2f6b29916c09671a0443c2cc60772b73 Mon Sep 17 00:00:00 2001 From: maximlt Date: Tue, 21 Jan 2025 22:57:36 +0100 Subject: [PATCH 11/14] skip networkx on macos and Python 3.12 --- doc/conftest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/conftest.py b/doc/conftest.py index 7ad65db45..a9a5a9a43 100644 --- a/doc/conftest.py +++ b/doc/conftest.py @@ -1,3 +1,5 @@ +import sys + from importlib.util import find_spec import dask @@ -10,7 +12,11 @@ 'user_guide/Streaming.ipynb', ] -if not find_spec('pygraphviz'): +# On MacOs, Python 3.12, got the following error running this: +# `pos = layout(G)` +# => OSError: Format: "dot" not recognized. No formats found. +# Fixed locally by running `dot -c` +if not find_spec('pygraphviz') or (sys.platform == 'darwin' and sys.version_info[:2] == (3, 12)): collect_ignore_glob += [ 'user_guide/NetworkX.ipynb', ] From c2a94aa82a0d3efb028c3c90a961004bdc38729f Mon Sep 17 00:00:00 2001 From: maximlt Date: Tue, 21 Jan 2025 23:06:44 +0100 Subject: [PATCH 12/14] attempt to skip some faulty test --- hvplot/tests/testpatch.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/hvplot/tests/testpatch.py b/hvplot/tests/testpatch.py index 9ed865ae6..1a74eb3fb 100644 --- a/hvplot/tests/testpatch.py +++ b/hvplot/tests/testpatch.py @@ -2,10 +2,13 @@ Tests patching of supported libraries """ +import sys + from unittest import TestCase, SkipTest import numpy as np import pandas as pd +import pytest from hvplot.plotting import hvPlotTabular, hvPlot @@ -117,6 +120,8 @@ def test_polars_series_patched(self): pseries = pl.Series([0, 1, 2]) self.assertIsInstance(pseries.hvplot, hvPlotTabular) + # stack overflow error on the CI + @pytest.mark.skipif(sys.platform == 'win32' and sys.version[:2] == (3, 9)) def test_polars_dataframe_patched(self): import polars as pl From 9ad76d48a4a900981d7b2378bc14d28acf1474e2 Mon Sep 17 00:00:00 2001 From: maximlt Date: Tue, 21 Jan 2025 23:11:26 +0100 Subject: [PATCH 13/14] add reason --- hvplot/tests/testpatch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hvplot/tests/testpatch.py b/hvplot/tests/testpatch.py index 1a74eb3fb..1b3c700e6 100644 --- a/hvplot/tests/testpatch.py +++ b/hvplot/tests/testpatch.py @@ -120,8 +120,10 @@ def test_polars_series_patched(self): pseries = pl.Series([0, 1, 2]) self.assertIsInstance(pseries.hvplot, hvPlotTabular) - # stack overflow error on the CI - @pytest.mark.skipif(sys.platform == 'win32' and sys.version[:2] == (3, 9)) + @pytest.mark.skipif( + sys.platform == 'win32' and sys.version[:2] == (3, 9), + reason='stack overflow error on the CI', + ) def test_polars_dataframe_patched(self): import polars as pl From 51440c8cb08d26f1dcd84d87199283597d8b2dc1 Mon Sep 17 00:00:00 2001 From: maximlt Date: Wed, 22 Jan 2025 00:49:03 +0100 Subject: [PATCH 14/14] fix tests --- hvplot/tests/testpatch.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hvplot/tests/testpatch.py b/hvplot/tests/testpatch.py index 1b3c700e6..2b54da02c 100644 --- a/hvplot/tests/testpatch.py +++ b/hvplot/tests/testpatch.py @@ -8,7 +8,6 @@ import numpy as np import pandas as pd -import pytest from hvplot.plotting import hvPlotTabular, hvPlot @@ -108,6 +107,8 @@ def test_streamz_seriess_patched(self): class TestPatchPolars(TestCase): def setUp(self): + if sys.platform == 'win32' and sys.version_info[:2] == (3, 9): + raise SkipTest('stack overflow error') try: import polars as pl # noqa except ImportError: @@ -120,10 +121,6 @@ def test_polars_series_patched(self): pseries = pl.Series([0, 1, 2]) self.assertIsInstance(pseries.hvplot, hvPlotTabular) - @pytest.mark.skipif( - sys.platform == 'win32' and sys.version[:2] == (3, 9), - reason='stack overflow error on the CI', - ) def test_polars_dataframe_patched(self): import polars as pl