enforce wheel size limits and README formatting in CI, put a ceiling …

…on Cython dependency (#2490) Contributes to rapidsai/build-planning#110 Proposes adding 2 types of validation on wheels in CI, to ensure we continue to produce wheels that are suitable for PyPI. * checks on wheel size (compressed), - *to be sure they're under PyPI limits* - *and to prompt discussion on PRs that significantly increase wheel sizes* * checks on README formatting - *to ensure they'll render properly as the PyPI project homepages* - *e.g. like how https://github.com/scikit-learn/scikit-learn/blob/main/README.rst becomes https://pypi.org/project/scikit-learn/* Also puts a ceiling on Cython to its latest stable release (`<=3.0.11`), to fix #2490 (comment). Work to relax that is tracked in (#2491). Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) URL: #2490
rapidsai · Nov 14, 2024 · a6712ba · a6712ba
1 parent 615447f
commit a6712ba
Show file tree

Hide file tree

Showing 16 changed files with 58 additions and 15 deletions.
diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh
@@ -3,6 +3,8 @@
 
 set -euo pipefail
 
+package_dir="python/pylibraft"
+
 case "${RAPIDS_CUDA_VERSION}" in
   12.*)
     EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON"
@@ -15,4 +17,5 @@ esac
 # Set up skbuild options. Enable sccache in skbuild config options
 export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}"
 
-ci/build_wheel.sh pylibraft python/pylibraft
+ci/build_wheel.sh pylibraft ${package_dir}
+ci/validate_wheel.sh ${package_dir} final_dist
diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh
@@ -3,7 +3,10 @@
 
 set -euo pipefail
 
+package_dir="python/raft-dask"
+
 # Set up skbuild options. Enable sccache in skbuild config options
 export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF"
 
-ci/build_wheel.sh raft-dask python/raft-dask
+ci/build_wheel.sh raft-dask ${package_dir}
+ci/validate_wheel.sh ${package_dir} final_dist
diff --git a/ci/validate_wheel.sh b/ci/validate_wheel.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+set -euo pipefail
+
+package_dir=$1
+wheel_dir_relative_path=$2
+
+cd "${package_dir}"
+
+rapids-logger "validate packages with 'pydistcheck'"
+
+pydistcheck \
+    --inspect \
+    "$(echo ${wheel_dir_relative_path}/*.whl)"
+
+rapids-logger "validate packages with 'twine'"
+
+twine check \
+    --strict \
+    "$(echo ${wheel_dir_relative_path}/*.whl)"
diff --git a/conda/environments/all_cuda-118_arch-aarch64.yaml b/conda/environments/all_cuda-118_arch-aarch64.yaml
@@ -19,7 +19,7 @@ dependencies:
 - cudatoolkit
 - cupy>=12.0.0
 - cxx-compiler
-- cython>=3.0.0
+- cython>=3.0.0,<3.1.0a0
 - dask-cuda==24.12.*,>=0.0.0a0
 - distributed-ucxx==0.41.*,>=0.0.0a0
 - doxygen>=1.8.20

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -19,7 +19,7 @@ dependencies:
 - cudatoolkit
 - cupy>=12.0.0
 - cxx-compiler
-- cython>=3.0.0
+- cython>=3.0.0,<3.1.0a0
 - dask-cuda==24.12.*,>=0.0.0a0
 - distributed-ucxx==0.41.*,>=0.0.0a0
 - doxygen>=1.8.20

diff --git a/conda/environments/all_cuda-125_arch-aarch64.yaml b/conda/environments/all_cuda-125_arch-aarch64.yaml
@@ -20,7 +20,7 @@ dependencies:
 - cuda-version=12.5
 - cupy>=12.0.0
 - cxx-compiler
-- cython>=3.0.0
+- cython>=3.0.0,<3.1.0a0
 - dask-cuda==24.12.*,>=0.0.0a0
 - distributed-ucxx==0.41.*,>=0.0.0a0
 - doxygen>=1.8.20

diff --git a/conda/environments/all_cuda-125_arch-x86_64.yaml b/conda/environments/all_cuda-125_arch-x86_64.yaml
@@ -20,7 +20,7 @@ dependencies:
 - cuda-version=12.5
 - cupy>=12.0.0
 - cxx-compiler
-- cython>=3.0.0
+- cython>=3.0.0,<3.1.0a0
 - dask-cuda==24.12.*,>=0.0.0a0
 - distributed-ucxx==0.41.*,>=0.0.0a0
 - doxygen>=1.8.20

diff --git a/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-118_arch-aarch64.yaml
@@ -17,7 +17,7 @@ dependencies:
 - cuda-version=11.8
 - cudatoolkit
 - cxx-compiler
-- cython>=3.0.0
+- cython>=3.0.0,<3.1.0a0
 - gcc_linux-aarch64=11.*
 - glog>=0.6.0
 - h5py>=3.8.0

diff --git a/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-118_arch-x86_64.yaml
@@ -17,7 +17,7 @@ dependencies:
 - cuda-version=11.8
 - cudatoolkit
 - cxx-compiler
-- cython>=3.0.0
+- cython>=3.0.0,<3.1.0a0
 - gcc_linux-64=11.*
 - glog>=0.6.0
 - h5py>=3.8.0

diff --git a/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml b/conda/environments/bench_ann_cuda-120_arch-aarch64.yaml
@@ -18,7 +18,7 @@ dependencies:
 - cuda-profiler-api
 - cuda-version=12.0
 - cxx-compiler
-- cython>=3.0.0
+- cython>=3.0.0,<3.1.0a0
 - gcc_linux-aarch64=11.*
 - glog>=0.6.0
 - h5py>=3.8.0

diff --git a/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml b/conda/environments/bench_ann_cuda-120_arch-x86_64.yaml
@@ -18,7 +18,7 @@ dependencies:
 - cuda-profiler-api
 - cuda-version=12.0
 - cxx-compiler
-- cython>=3.0.0
+- cython>=3.0.0,<3.1.0a0
 - gcc_linux-64=11.*
 - glog>=0.6.0
 - h5py>=3.8.0

diff --git a/conda/recipes/pylibraft/meta.yaml b/conda/recipes/pylibraft/meta.yaml
@@ -50,7 +50,7 @@ requirements:
     - cuda-cudart-dev
     {% endif %}
     - cuda-version ={{ cuda_version }}
-    - cython >=3.0.0
+    - cython >=3.0.0,<3.1.0a0
     - libraft {{ version }}
     - libraft-headers {{ version }}
     - python x.x

diff --git a/conda/recipes/raft-dask/meta.yaml b/conda/recipes/raft-dask/meta.yaml
@@ -50,7 +50,7 @@ requirements:
     - cuda-cudart-dev
     {% endif %}
     - cuda-version ={{ cuda_version }}
-    - cython >=3.0.0
+    - cython >=3.0.0,<3.1.0a0
     - nccl {{ nccl_version }}
     - pylibraft {{ version }}
     - python x.x

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -164,7 +164,7 @@ dependencies:
       - output_types: [conda, requirements, pyproject]
         packages:
           - &cmake_ver cmake>=3.26.4,!=3.30.0
-          - cython>=3.0.0
+          - cython>=3.0.0,<3.1.0a0
           - ninja
       - output_types: [conda]
         packages:

diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml
@@ -123,13 +123,21 @@ build-backend = "scikit_build_core.build"
 requires = [
     "cmake>=3.26.4,!=3.30.0",
     "cuda-python",
-    "cython>=3.0.0",
+    "cython>=3.0.0,<3.1.0a0",
     "ninja",
     "rmm==24.12.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 dependencies-file = "../../dependencies.yaml"
 matrix-entry = "cuda_suffixed=true;use_cuda_wheels=true"
 
+[tool.pydistcheck]
+select = [
+    "distro-too-large-compressed",
+]
+
+# detect when package size grows significantly
+max_allowed_size_compressed = '825M'
+
 [tool.pytest.ini_options]
 filterwarnings = [
     "error",

diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml
@@ -120,9 +120,17 @@ regex = "(?P<value>.*)"
 build-backend = "scikit_build_core.build"
 requires = [
     "cmake>=3.26.4,!=3.30.0",
-    "cython>=3.0.0",
+    "cython>=3.0.0,<3.1.0a0",
     "libucx==1.15.0",
     "ninja",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 dependencies-file = "../../dependencies.yaml"
 matrix-entry = "cuda_suffixed=true"
+
+[tool.pydistcheck]
+select = [
+    "distro-too-large-compressed",
+]
+
+# detect when package size grows significantly
+max_allowed_size_compressed = '300M'