From d650fa78c6b0a72faec1974baca8ce9afeb1e82d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 17 Apr 2024 09:01:27 -0700 Subject: [PATCH 01/76] WIP: add wheel output --- .github/workflows/pr.yaml | 7 +++++++ ci/build_wheel_python.sh | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100755 ci/build_wheel_python.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2e38cf9245..bab979ef57 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,6 +19,7 @@ jobs: - conda-python-tests - docs-build - devcontainer + - wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.06 checks: @@ -68,3 +69,9 @@ jobs: sccache -z; build-all --verbose; sccache -s; + wheel-build-python: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + build_type: pull-request + script: ci/build_wheel_python.sh diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh new file mode 100755 index 0000000000..26e27c81de --- /dev/null +++ b/ci/build_wheel_python.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="kvikio" +package_dir="python" + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +pyproject_file="${package_dir}/pyproject.toml" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" + +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} +fi + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist \ No newline at end of file From 1faa5173d845aa15df13776175a7db359c8fdaae Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 17 Apr 2024 10:26:04 -0700 Subject: [PATCH 02/76] reorganize python folder to allow for libkvikio wheel --- .github/workflows/pr.yaml | 9 +++++ VERSION | 2 +- ci/build_wheel_cpp.sh | 38 +++++++++++++++++++ ci/build_wheel_python.sh | 6 ++- conda/recipes/kvikio/meta.yaml | 2 +- dependencies.yaml | 21 +++++++++- python/README.md | 1 - python/{ => kvikio}/.coveragerc | 0 python/{ => kvikio}/.gitattributes | 0 python/{ => kvikio}/CMakeLists.txt | 9 +---- python/kvikio/README.md | 1 + python/kvikio/VERSION | 1 - .../{ => kvikio}/benchmarks/single-node-io.py | 0 python/{ => kvikio}/benchmarks/zarr-io.py | 0 python/{ => kvikio}/cli/gpu_compressor.py | 0 python/{ => kvikio}/cmake/CMakeLists.txt | 0 .../cmake/thirdparty/get_nvcomp.cmake | 0 python/{ => kvikio}/examples/hello_world.py | 0 .../{ => kvikio}/examples/zarr_cupy_nvcomp.py | 0 python/kvikio/kvikio/VERSION | 1 + python/kvikio/{ => kvikio}/__init__.py | 0 .../kvikio/{ => kvikio}/_lib/CMakeLists.txt | 0 python/kvikio/{ => kvikio}/_lib/__init__.pxd | 0 python/kvikio/{ => kvikio}/_lib/__init__.py | 0 python/kvikio/{ => kvikio}/_lib/arr.pxd | 0 python/kvikio/{ => kvikio}/_lib/arr.pyi | 0 python/kvikio/{ => kvikio}/_lib/arr.pyx | 0 .../{ => kvikio}/_lib/kvikio_cxx_api.pxd | 0 python/kvikio/{ => kvikio}/_lib/libkvikio.pyx | 0 python/kvikio/{ => kvikio}/_lib/libnvcomp.pyx | 0 .../kvikio/{ => kvikio}/_lib/libnvcomp_ll.pyx | 0 .../{ => kvikio}/_lib/nvcomp_cxx_api.pxd | 0 .../{ => kvikio}/_lib/nvcomp_ll_cxx_api.pxd | 0 python/kvikio/{ => kvikio}/_version.py | 2 +- python/kvikio/{ => kvikio}/cufile.py | 0 python/kvikio/{ => kvikio}/defaults.py | 0 python/kvikio/{ => kvikio}/numcodecs.py | 0 python/kvikio/{ => kvikio}/numpy.py | 0 python/kvikio/{ => kvikio}/nvcomp.py | 0 python/kvikio/{ => kvikio}/nvcomp_codec.py | 0 python/kvikio/{ => kvikio}/zarr.py | 0 python/{ => kvikio}/pyproject.toml | 3 +- python/{ => kvikio}/tests/conftest.py | 0 python/{ => kvikio}/tests/test_basic_io.py | 0 python/{ => kvikio}/tests/test_benchmarks.py | 0 python/{ => kvikio}/tests/test_defaults.py | 0 python/{ => kvikio}/tests/test_examples.py | 0 python/{ => kvikio}/tests/test_numpy.py | 0 python/{ => kvikio}/tests/test_nvcomp.py | 0 .../{ => kvikio}/tests/test_nvcomp_codec.py | 0 python/{ => kvikio}/tests/test_zarr.py | 0 python/libkvikio/CMakeLists.txt | 28 ++++++++++++++ python/libkvikio/LICENSE | 1 + python/libkvikio/README.md | 1 + python/libkvikio/libkvikio/VERSION | 1 + python/libkvikio/libkvikio/__init__.py | 17 +++++++++ python/libkvikio/libkvikio/_version.py | 21 ++++++++++ 57 files changed, 147 insertions(+), 18 deletions(-) create mode 100644 ci/build_wheel_cpp.sh delete mode 120000 python/README.md rename python/{ => kvikio}/.coveragerc (100%) rename python/{ => kvikio}/.gitattributes (100%) rename python/{ => kvikio}/CMakeLists.txt (86%) create mode 120000 python/kvikio/README.md delete mode 120000 python/kvikio/VERSION rename python/{ => kvikio}/benchmarks/single-node-io.py (100%) rename python/{ => kvikio}/benchmarks/zarr-io.py (100%) rename python/{ => kvikio}/cli/gpu_compressor.py (100%) rename python/{ => kvikio}/cmake/CMakeLists.txt (100%) rename python/{ => kvikio}/cmake/thirdparty/get_nvcomp.cmake (100%) rename python/{ => kvikio}/examples/hello_world.py (100%) rename python/{ => kvikio}/examples/zarr_cupy_nvcomp.py (100%) create mode 120000 python/kvikio/kvikio/VERSION rename python/kvikio/{ => kvikio}/__init__.py (100%) rename python/kvikio/{ => kvikio}/_lib/CMakeLists.txt (100%) rename python/kvikio/{ => kvikio}/_lib/__init__.pxd (100%) rename python/kvikio/{ => kvikio}/_lib/__init__.py (100%) rename python/kvikio/{ => kvikio}/_lib/arr.pxd (100%) rename python/kvikio/{ => kvikio}/_lib/arr.pyi (100%) rename python/kvikio/{ => kvikio}/_lib/arr.pyx (100%) rename python/kvikio/{ => kvikio}/_lib/kvikio_cxx_api.pxd (100%) rename python/kvikio/{ => kvikio}/_lib/libkvikio.pyx (100%) rename python/kvikio/{ => kvikio}/_lib/libnvcomp.pyx (100%) rename python/kvikio/{ => kvikio}/_lib/libnvcomp_ll.pyx (100%) rename python/kvikio/{ => kvikio}/_lib/nvcomp_cxx_api.pxd (100%) rename python/kvikio/{ => kvikio}/_lib/nvcomp_ll_cxx_api.pxd (100%) rename python/kvikio/{ => kvikio}/_version.py (92%) rename python/kvikio/{ => kvikio}/cufile.py (100%) rename python/kvikio/{ => kvikio}/defaults.py (100%) rename python/kvikio/{ => kvikio}/numcodecs.py (100%) rename python/kvikio/{ => kvikio}/numpy.py (100%) rename python/kvikio/{ => kvikio}/nvcomp.py (100%) rename python/kvikio/{ => kvikio}/nvcomp_codec.py (100%) rename python/kvikio/{ => kvikio}/zarr.py (100%) rename python/{ => kvikio}/pyproject.toml (98%) rename python/{ => kvikio}/tests/conftest.py (100%) rename python/{ => kvikio}/tests/test_basic_io.py (100%) rename python/{ => kvikio}/tests/test_benchmarks.py (100%) rename python/{ => kvikio}/tests/test_defaults.py (100%) rename python/{ => kvikio}/tests/test_examples.py (100%) rename python/{ => kvikio}/tests/test_numpy.py (100%) rename python/{ => kvikio}/tests/test_nvcomp.py (100%) rename python/{ => kvikio}/tests/test_nvcomp_codec.py (100%) rename python/{ => kvikio}/tests/test_zarr.py (100%) create mode 100644 python/libkvikio/CMakeLists.txt create mode 120000 python/libkvikio/LICENSE create mode 120000 python/libkvikio/README.md create mode 120000 python/libkvikio/libkvikio/VERSION create mode 100644 python/libkvikio/libkvikio/__init__.py create mode 100644 python/libkvikio/libkvikio/_version.py diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index bab979ef57..1861608660 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,6 +19,7 @@ jobs: - conda-python-tests - docs-build - devcontainer + - wheel-build-cpp - wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.06 @@ -69,7 +70,15 @@ jobs: sccache -z; build-all --verbose; sccache -s; + wheel-build-cpp: + secrets: inherit + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + build_type: pull-request + script: ci/build_wheel_cpp.sh wheel-build-python: + needs: wheel-build-cpp secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: diff --git a/VERSION b/VERSION index 0bff6981a3..cb16728b39 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.06.00 +24.06.00a11 diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh new file mode 100644 index 0000000000..a8e270cce6 --- /dev/null +++ b/ci/build_wheel_cpp.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Copyright (c) 2023, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="kvikio" +package_dir="python/libkvikio" + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +pyproject_file="${package_dir}/pyproject.toml" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" + +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} +fi + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist \ No newline at end of file diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 26e27c81de..ee3f50e502 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -4,7 +4,7 @@ set -euo pipefail package_name="kvikio" -package_dir="python" +package_dir="python/kvikio" source rapids-configure-sccache source rapids-date-string @@ -28,9 +28,11 @@ if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} fi +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) + cd "${package_dir}" -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE} mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* diff --git a/conda/recipes/kvikio/meta.yaml b/conda/recipes/kvikio/meta.yaml index 36f4c06bc1..3d01397113 100644 --- a/conda/recipes/kvikio/meta.yaml +++ b/conda/recipes/kvikio/meta.yaml @@ -33,7 +33,7 @@ build: - SCCACHE_S3_USE_SSL - SCCACHE_S3_NO_CREDENTIALS script: - - cd python + - cd python/kvikio - python -m pip install . -vv ignore_run_exports_from: {% if cuda_major == "11" %} diff --git a/dependencies.yaml b/dependencies.yaml index 1e17c924f0..71cbda52ad 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -38,19 +38,31 @@ files: - py_version py_build: output: pyproject - pyproject_dir: python + pyproject_dir: python/kvikio extras: table: build-system includes: - build py_run: output: pyproject - pyproject_dir: python + pyproject_dir: python/kvikio extras: table: project includes: - depends_on_cupy - run + py_wheel_cpp: + output: pyproject/libkvikio + extras: + table: project + includes: + - build + py_wheel_python: + output: pyproject/kvikio + extras: + table: project + includes: + - build py_optional_test: output: pyproject pyproject_dir: python @@ -208,6 +220,11 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} + wheel_lib_dep: + common: + - output_types: pyproject + packages: + - libkvikio==24.6.* docs: common: - output_types: [conda, requirements] diff --git a/python/README.md b/python/README.md deleted file mode 120000 index 32d46ee883..0000000000 --- a/python/README.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/python/.coveragerc b/python/kvikio/.coveragerc similarity index 100% rename from python/.coveragerc rename to python/kvikio/.coveragerc diff --git a/python/.gitattributes b/python/kvikio/.gitattributes similarity index 100% rename from python/.gitattributes rename to python/kvikio/.gitattributes diff --git a/python/CMakeLists.txt b/python/kvikio/CMakeLists.txt similarity index 86% rename from python/CMakeLists.txt rename to python/kvikio/CMakeLists.txt index 039daf7ce9..22436c0839 100644 --- a/python/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -26,18 +26,11 @@ project( LANGUAGES CXX CUDA ) -option(FIND_KVIKIO_CPP - "Search for existing KVIKIO C++ installations before defaulting to local files" OFF -) # TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile include(../cpp/cmake/Modules/FindcuFile.cmake) -if(FIND_KVIKIO_CPP) - find_package(KvikIO "${RAPIDS_VERSION}") -else() - set(KvikIO_FOUND OFF) -endif() +find_package(KvikIO "${RAPIDS_VERSION}") find_package(CUDAToolkit REQUIRED) diff --git a/python/kvikio/README.md b/python/kvikio/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/kvikio/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/kvikio/VERSION b/python/kvikio/VERSION deleted file mode 120000 index 558194c5a5..0000000000 --- a/python/kvikio/VERSION +++ /dev/null @@ -1 +0,0 @@ -../../VERSION \ No newline at end of file diff --git a/python/benchmarks/single-node-io.py b/python/kvikio/benchmarks/single-node-io.py similarity index 100% rename from python/benchmarks/single-node-io.py rename to python/kvikio/benchmarks/single-node-io.py diff --git a/python/benchmarks/zarr-io.py b/python/kvikio/benchmarks/zarr-io.py similarity index 100% rename from python/benchmarks/zarr-io.py rename to python/kvikio/benchmarks/zarr-io.py diff --git a/python/cli/gpu_compressor.py b/python/kvikio/cli/gpu_compressor.py similarity index 100% rename from python/cli/gpu_compressor.py rename to python/kvikio/cli/gpu_compressor.py diff --git a/python/cmake/CMakeLists.txt b/python/kvikio/cmake/CMakeLists.txt similarity index 100% rename from python/cmake/CMakeLists.txt rename to python/kvikio/cmake/CMakeLists.txt diff --git a/python/cmake/thirdparty/get_nvcomp.cmake b/python/kvikio/cmake/thirdparty/get_nvcomp.cmake similarity index 100% rename from python/cmake/thirdparty/get_nvcomp.cmake rename to python/kvikio/cmake/thirdparty/get_nvcomp.cmake diff --git a/python/examples/hello_world.py b/python/kvikio/examples/hello_world.py similarity index 100% rename from python/examples/hello_world.py rename to python/kvikio/examples/hello_world.py diff --git a/python/examples/zarr_cupy_nvcomp.py b/python/kvikio/examples/zarr_cupy_nvcomp.py similarity index 100% rename from python/examples/zarr_cupy_nvcomp.py rename to python/kvikio/examples/zarr_cupy_nvcomp.py diff --git a/python/kvikio/kvikio/VERSION b/python/kvikio/kvikio/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/kvikio/kvikio/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/kvikio/__init__.py b/python/kvikio/kvikio/__init__.py similarity index 100% rename from python/kvikio/__init__.py rename to python/kvikio/kvikio/__init__.py diff --git a/python/kvikio/_lib/CMakeLists.txt b/python/kvikio/kvikio/_lib/CMakeLists.txt similarity index 100% rename from python/kvikio/_lib/CMakeLists.txt rename to python/kvikio/kvikio/_lib/CMakeLists.txt diff --git a/python/kvikio/_lib/__init__.pxd b/python/kvikio/kvikio/_lib/__init__.pxd similarity index 100% rename from python/kvikio/_lib/__init__.pxd rename to python/kvikio/kvikio/_lib/__init__.pxd diff --git a/python/kvikio/_lib/__init__.py b/python/kvikio/kvikio/_lib/__init__.py similarity index 100% rename from python/kvikio/_lib/__init__.py rename to python/kvikio/kvikio/_lib/__init__.py diff --git a/python/kvikio/_lib/arr.pxd b/python/kvikio/kvikio/_lib/arr.pxd similarity index 100% rename from python/kvikio/_lib/arr.pxd rename to python/kvikio/kvikio/_lib/arr.pxd diff --git a/python/kvikio/_lib/arr.pyi b/python/kvikio/kvikio/_lib/arr.pyi similarity index 100% rename from python/kvikio/_lib/arr.pyi rename to python/kvikio/kvikio/_lib/arr.pyi diff --git a/python/kvikio/_lib/arr.pyx b/python/kvikio/kvikio/_lib/arr.pyx similarity index 100% rename from python/kvikio/_lib/arr.pyx rename to python/kvikio/kvikio/_lib/arr.pyx diff --git a/python/kvikio/_lib/kvikio_cxx_api.pxd b/python/kvikio/kvikio/_lib/kvikio_cxx_api.pxd similarity index 100% rename from python/kvikio/_lib/kvikio_cxx_api.pxd rename to python/kvikio/kvikio/_lib/kvikio_cxx_api.pxd diff --git a/python/kvikio/_lib/libkvikio.pyx b/python/kvikio/kvikio/_lib/libkvikio.pyx similarity index 100% rename from python/kvikio/_lib/libkvikio.pyx rename to python/kvikio/kvikio/_lib/libkvikio.pyx diff --git a/python/kvikio/_lib/libnvcomp.pyx b/python/kvikio/kvikio/_lib/libnvcomp.pyx similarity index 100% rename from python/kvikio/_lib/libnvcomp.pyx rename to python/kvikio/kvikio/_lib/libnvcomp.pyx diff --git a/python/kvikio/_lib/libnvcomp_ll.pyx b/python/kvikio/kvikio/_lib/libnvcomp_ll.pyx similarity index 100% rename from python/kvikio/_lib/libnvcomp_ll.pyx rename to python/kvikio/kvikio/_lib/libnvcomp_ll.pyx diff --git a/python/kvikio/_lib/nvcomp_cxx_api.pxd b/python/kvikio/kvikio/_lib/nvcomp_cxx_api.pxd similarity index 100% rename from python/kvikio/_lib/nvcomp_cxx_api.pxd rename to python/kvikio/kvikio/_lib/nvcomp_cxx_api.pxd diff --git a/python/kvikio/_lib/nvcomp_ll_cxx_api.pxd b/python/kvikio/kvikio/_lib/nvcomp_ll_cxx_api.pxd similarity index 100% rename from python/kvikio/_lib/nvcomp_ll_cxx_api.pxd rename to python/kvikio/kvikio/_lib/nvcomp_ll_cxx_api.pxd diff --git a/python/kvikio/_version.py b/python/kvikio/kvikio/_version.py similarity index 92% rename from python/kvikio/_version.py rename to python/kvikio/kvikio/_version.py index 82fc478d5c..75cad4e207 100644 --- a/python/kvikio/_version.py +++ b/python/kvikio/kvikio/_version.py @@ -17,4 +17,4 @@ __version__ = ( importlib.resources.files("kvikio").joinpath("VERSION").read_text().strip() ) -__git_commit__ = "" +__git_commit__ = "7b0231c8241164c8a0511cd737e7098cb7ccea3e" diff --git a/python/kvikio/cufile.py b/python/kvikio/kvikio/cufile.py similarity index 100% rename from python/kvikio/cufile.py rename to python/kvikio/kvikio/cufile.py diff --git a/python/kvikio/defaults.py b/python/kvikio/kvikio/defaults.py similarity index 100% rename from python/kvikio/defaults.py rename to python/kvikio/kvikio/defaults.py diff --git a/python/kvikio/numcodecs.py b/python/kvikio/kvikio/numcodecs.py similarity index 100% rename from python/kvikio/numcodecs.py rename to python/kvikio/kvikio/numcodecs.py diff --git a/python/kvikio/numpy.py b/python/kvikio/kvikio/numpy.py similarity index 100% rename from python/kvikio/numpy.py rename to python/kvikio/kvikio/numpy.py diff --git a/python/kvikio/nvcomp.py b/python/kvikio/kvikio/nvcomp.py similarity index 100% rename from python/kvikio/nvcomp.py rename to python/kvikio/kvikio/nvcomp.py diff --git a/python/kvikio/nvcomp_codec.py b/python/kvikio/kvikio/nvcomp_codec.py similarity index 100% rename from python/kvikio/nvcomp_codec.py rename to python/kvikio/kvikio/nvcomp_codec.py diff --git a/python/kvikio/zarr.py b/python/kvikio/kvikio/zarr.py similarity index 100% rename from python/kvikio/zarr.py rename to python/kvikio/kvikio/zarr.py diff --git a/python/pyproject.toml b/python/kvikio/pyproject.toml similarity index 98% rename from python/pyproject.toml rename to python/kvikio/pyproject.toml index 291b57f2c4..32069c65b8 100644 --- a/python/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -11,7 +11,7 @@ requires = [ ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. [project] -name = "kvikio" +name = "kvikio-cu11" dynamic = ["version"] description = "KvikIO - GPUDirect Storage" readme = { file = "README.md", content-type = "text/markdown" } @@ -24,6 +24,7 @@ dependencies = [ "cupy-cuda11x>=12.0.0", "numcodecs <0.12.0", "numpy>=1.23,<2.0a0", + "libkvikio==24.06.*", "packaging", "zarr", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/tests/conftest.py b/python/kvikio/tests/conftest.py similarity index 100% rename from python/tests/conftest.py rename to python/kvikio/tests/conftest.py diff --git a/python/tests/test_basic_io.py b/python/kvikio/tests/test_basic_io.py similarity index 100% rename from python/tests/test_basic_io.py rename to python/kvikio/tests/test_basic_io.py diff --git a/python/tests/test_benchmarks.py b/python/kvikio/tests/test_benchmarks.py similarity index 100% rename from python/tests/test_benchmarks.py rename to python/kvikio/tests/test_benchmarks.py diff --git a/python/tests/test_defaults.py b/python/kvikio/tests/test_defaults.py similarity index 100% rename from python/tests/test_defaults.py rename to python/kvikio/tests/test_defaults.py diff --git a/python/tests/test_examples.py b/python/kvikio/tests/test_examples.py similarity index 100% rename from python/tests/test_examples.py rename to python/kvikio/tests/test_examples.py diff --git a/python/tests/test_numpy.py b/python/kvikio/tests/test_numpy.py similarity index 100% rename from python/tests/test_numpy.py rename to python/kvikio/tests/test_numpy.py diff --git a/python/tests/test_nvcomp.py b/python/kvikio/tests/test_nvcomp.py similarity index 100% rename from python/tests/test_nvcomp.py rename to python/kvikio/tests/test_nvcomp.py diff --git a/python/tests/test_nvcomp_codec.py b/python/kvikio/tests/test_nvcomp_codec.py similarity index 100% rename from python/tests/test_nvcomp_codec.py rename to python/kvikio/tests/test_nvcomp_codec.py diff --git a/python/tests/test_zarr.py b/python/kvikio/tests/test_zarr.py similarity index 100% rename from python/tests/test_zarr.py rename to python/kvikio/tests/test_zarr.py diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt new file mode 100644 index 0000000000..8cb4189eb3 --- /dev/null +++ b/python/libkvikio/CMakeLists.txt @@ -0,0 +1,28 @@ +# ============================================================================= +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +project( + libkvikio-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX) + +set(BUILD_TESTS OFF) +set(BUILD_BENCHMARKS OFF) +set(CUDA_STATIC_RUNTIME ON) + +add_subdirectory(../../cpp kvikio-cpp) \ No newline at end of file diff --git a/python/libkvikio/LICENSE b/python/libkvikio/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libkvikio/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libkvikio/README.md b/python/libkvikio/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libkvikio/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libkvikio/libkvikio/VERSION b/python/libkvikio/libkvikio/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libkvikio/libkvikio/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libkvikio/libkvikio/__init__.py b/python/libkvikio/libkvikio/__init__.py new file mode 100644 index 0000000000..e046016b48 --- /dev/null +++ b/python/libkvikio/libkvikio/__init__.py @@ -0,0 +1,17 @@ + +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libkvikio._version import __git_commit__, __version__ + diff --git a/python/libkvikio/libkvikio/_version.py b/python/libkvikio/libkvikio/_version.py new file mode 100644 index 0000000000..31548c3a28 --- /dev/null +++ b/python/libkvikio/libkvikio/_version.py @@ -0,0 +1,21 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files("libkvikio").joinpath("VERSION").read_text().strip() +) +__git_commit__ = "" + From ea1992bd22d5e68a6e91149aae211c6a969cdfee Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 17 Apr 2024 12:02:30 -0700 Subject: [PATCH 03/76] fix matrix_filter in the wrong place --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1861608660..55578b4bd3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -72,9 +72,9 @@ jobs: sccache -s; wheel-build-cpp: secrets: inherit - matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request script: ci/build_wheel_cpp.sh wheel-build-python: From 3b35c05ae58bd52095b4794ba089dec28a18695a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 17 Apr 2024 12:21:17 -0700 Subject: [PATCH 04/76] fix executability and add missing pyproject.toml --- ci/build_wheel_cpp.sh | 0 python/libkvikio/pyproject.toml | 121 ++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+) mode change 100644 => 100755 ci/build_wheel_cpp.sh create mode 100644 python/libkvikio/pyproject.toml diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh old mode 100644 new mode 100755 diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml new file mode 100644 index 0000000000..897996409a --- /dev/null +++ b/python/libkvikio/pyproject.toml @@ -0,0 +1,121 @@ +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# See file LICENSE for terms. + +[build-system] +build-backend = "scikit_build_core.build" +requires = [ + "cmake>=3.26.4", + "cython>=3.0.0", + "ninja", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libkvikio" +dynamic = ["version"] +description = "KvikIO - GPUDirect Storage" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.9" +dependencies = [ + "cupy-cuda11x>=12.0.0", + "numcodecs <0.12.0", + "numpy>=1.23,<2.0a0", + "packaging", + "zarr", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] + +[project.optional-dependencies] +test = [ + "cuda-python>=11.7.1,<12.0a0", + "dask>=2022.05.2", + "pytest", + "pytest-cov", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/kvikio" + +[tool.black] +line-length = 88 +target-version = ["py39"] +include = '\.py?$' +exclude = ''' +/( + thirdparty | + \.eggs | + \.git | + \.hg | + \.mypy_cache | + \.tox | + \.venv | + _build | + buck-out | + build | + dist | + _skbuild +)/ +''' + +[tool.isort] +line_length = 88 +multi_line_output = 3 +include_trailing_comma = true +force_grid_wrap = 0 +combine_as_imports = true +order_by_type = true +known_first_party = [ + "kvikio", +] +default_section = "THIRDPARTY" +sections = [ + "FUTURE", + "STDLIB", + "THIRDPARTY", + "FIRSTPARTY", + "LOCALFOLDER", +] +skip = [ + "thirdparty", + ".eggs", + ".git", + ".hg", + ".mypy_cache", + ".tox", + ".venv", + "_build", + "buck-out", + "build", + "dist", + "__init__.py", +] + +[tool.mypy] +ignore_missing_imports = true + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.exclude = ["*tests*"] +sdist.reproducible = true +wheel.packages = ["libkvikio"] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libkvikio/VERSION" +regex = "(?P.*)" From 3f9e1fe82f180f98dedc9f3007e08f9a7fc566c2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 17 Apr 2024 12:48:10 -0700 Subject: [PATCH 05/76] fix version file location --- ci/build_wheel_cpp.sh | 2 +- python/libkvikio/.gitattributes | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 python/libkvikio/.gitattributes diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index a8e270cce6..51fcb6d779 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -3,7 +3,7 @@ set -euo pipefail -package_name="kvikio" +package_name="libkvikio" package_dir="python/libkvikio" source rapids-configure-sccache diff --git a/python/libkvikio/.gitattributes b/python/libkvikio/.gitattributes new file mode 100644 index 0000000000..a2d7dee9e0 --- /dev/null +++ b/python/libkvikio/.gitattributes @@ -0,0 +1 @@ +libkvikio/_version.py export-subst From a3e25f14aec72d9bf73a604616c99f05b2e2a5ae Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 22 Apr 2024 12:58:56 -0700 Subject: [PATCH 06/76] fix missing lib files --- python/kvikio/kvikio/__init__.py | 11 ++++++ python/libkvikio/libkvikio/__init__.py | 2 +- python/libkvikio/libkvikio/load.py | 50 ++++++++++++++++++++++++++ rapids_config.cmake | 36 +++++++++++++++++++ 4 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 python/libkvikio/libkvikio/load.py create mode 100644 rapids_config.cmake diff --git a/python/kvikio/kvikio/__init__.py b/python/kvikio/kvikio/__init__.py index 5186df50f8..a280a370c3 100644 --- a/python/kvikio/kvikio/__init__.py +++ b/python/kvikio/kvikio/__init__.py @@ -1,6 +1,17 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. # See file LICENSE for terms. + +# If libcudf was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libkvikio +except ModuleNotFoundError: + pass +else: + libkvikio.load_library() + del libkvikio + from ._lib import libkvikio # type: ignore from ._version import __version__ # noqa: F401 from .cufile import CuFile # noqa: F401 diff --git a/python/libkvikio/libkvikio/__init__.py b/python/libkvikio/libkvikio/__init__.py index e046016b48..a837803a02 100644 --- a/python/libkvikio/libkvikio/__init__.py +++ b/python/libkvikio/libkvikio/__init__.py @@ -14,4 +14,4 @@ # limitations under the License. from libkvikio._version import __git_commit__, __version__ - +from libkvikio.load import load_library \ No newline at end of file diff --git a/python/libkvikio/libkvikio/load.py b/python/libkvikio/libkvikio/load.py new file mode 100644 index 0000000000..0b02b8b932 --- /dev/null +++ b/python/libkvikio/libkvikio/load.py @@ -0,0 +1,50 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + + +def load_library(): + # Dynamically load libkvikio.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libkvikio_lib = ctypes.CDLL("libkvikio.so", ctypes.RTLD_GLOBAL) + except OSError: + # load the nvcomp libraries from lib first, so that libcudf has them when we load it from lib64 + # TODO: if we put both of these in the same folder, I don't think we'd need this. + # for nvcomp_lib in ( + # "libnvcomp.so", + # "libnvcomp_gdeflate.so", + # "libnvcomp_bitcomp.so", + # ): + # _loaded_lib = ctypes.CDLL( + # # TODO: Do we always know it will be lib64? Should we consider + # # finding a way for CMake to export the path for us to find here? + # os.path.join(os.path.dirname(__file__), "lib", nvcomp_lib), + # ctypes.RTLD_GLOBAL, + # ) + libkvikio_lib = ctypes.CDLL( + # TODO: Do we always know it will be lib64? Should we consider + # finding a way for CMake to export the path for us to find here? + os.path.join(os.path.dirname(__file__), "lib64", "libkvikio.so"), + ctypes.RTLD_GLOBAL, + ) + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcudf was loaded from. + return libkvikio_lib diff --git a/rapids_config.cmake b/rapids_config.cmake new file mode 100644 index 0000000000..1413dd28d4 --- /dev/null +++ b/rapids_config.cmake @@ -0,0 +1,36 @@ +# ============================================================================= +# Copyright (c) 2018-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= +file(READ "${CMAKE_CURRENT_LIST_DIR}/VERSION" _rapids_version) +if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]]) + set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") + set(RAPIDS_VERSION_MINOR "${CMAKE_MATCH_2}") + set(RAPIDS_VERSION_PATCH "${CMAKE_MATCH_3}") + set(RAPIDS_VERSION_MAJOR_MINOR "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}") + set(RAPIDS_VERSION "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}.${RAPIDS_VERSION_PATCH}") +else() + string(REPLACE "\n" "\n " _rapids_version_formatted " ${_rapids_version}") + message( + FATAL_ERROR + "Could not determine RAPIDS version. Contents of VERSION file:\n${_rapids_version_formatted}" + ) +endif() + +if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") + file( + DOWNLOAD + "https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/RAPIDS.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake" + ) +endif() +include("${CMAKE_CURRENT_BINARY_DIR}/RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") From f8b55e0404a79ef99db3914697ce1ed269c866c8 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 23 Apr 2024 07:20:05 -0700 Subject: [PATCH 07/76] remove load.py (kvikio is header-only) --- python/libkvikio/CMakeLists.txt | 8 +++-- python/libkvikio/libkvikio/__init__.py | 1 - python/libkvikio/libkvikio/load.py | 50 -------------------------- 3 files changed, 5 insertions(+), 54 deletions(-) delete mode 100644 python/libkvikio/libkvikio/load.py diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 8cb4189eb3..84c53237a2 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -21,8 +21,10 @@ project( VERSION "${RAPIDS_VERSION}" LANGUAGES CXX) -set(BUILD_TESTS OFF) -set(BUILD_BENCHMARKS OFF) +set(KvikIO_BUILD_EXAMPLES OFF) +set(KvikIO_BUILD_TESTS OFF) set(CUDA_STATIC_RUNTIME ON) -add_subdirectory(../../cpp kvikio-cpp) \ No newline at end of file +set(CMAKE_FIND_DEBUG_MODE ON) + +add_subdirectory(../../cpp kvikio-cpp) diff --git a/python/libkvikio/libkvikio/__init__.py b/python/libkvikio/libkvikio/__init__.py index a837803a02..7371ab4c3d 100644 --- a/python/libkvikio/libkvikio/__init__.py +++ b/python/libkvikio/libkvikio/__init__.py @@ -14,4 +14,3 @@ # limitations under the License. from libkvikio._version import __git_commit__, __version__ -from libkvikio.load import load_library \ No newline at end of file diff --git a/python/libkvikio/libkvikio/load.py b/python/libkvikio/libkvikio/load.py deleted file mode 100644 index 0b02b8b932..0000000000 --- a/python/libkvikio/libkvikio/load.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) 2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import ctypes -import os - - -def load_library(): - # Dynamically load libkvikio.so. Prefer a system library if one is present to - # avoid clobbering symbols that other packages might expect, but if no - # other library is present use the one in the wheel. - try: - libkvikio_lib = ctypes.CDLL("libkvikio.so", ctypes.RTLD_GLOBAL) - except OSError: - # load the nvcomp libraries from lib first, so that libcudf has them when we load it from lib64 - # TODO: if we put both of these in the same folder, I don't think we'd need this. - # for nvcomp_lib in ( - # "libnvcomp.so", - # "libnvcomp_gdeflate.so", - # "libnvcomp_bitcomp.so", - # ): - # _loaded_lib = ctypes.CDLL( - # # TODO: Do we always know it will be lib64? Should we consider - # # finding a way for CMake to export the path for us to find here? - # os.path.join(os.path.dirname(__file__), "lib", nvcomp_lib), - # ctypes.RTLD_GLOBAL, - # ) - libkvikio_lib = ctypes.CDLL( - # TODO: Do we always know it will be lib64? Should we consider - # finding a way for CMake to export the path for us to find here? - os.path.join(os.path.dirname(__file__), "lib64", "libkvikio.so"), - ctypes.RTLD_GLOBAL, - ) - - # The caller almost never needs to do anything with this library, but no - # harm in offering the option since this object at least provides a handle - # to inspect where libcudf was loaded from. - return libkvikio_lib From 1c7ebd914b65d6654eb897ec419daefb7e4a779c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 30 Apr 2024 13:44:29 -0500 Subject: [PATCH 08/76] renaming workflows for consistency. Fix path for build.sh python --- .github/workflows/pr.yaml | 10 +++++----- build.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 55578b4bd3..5f68b77c53 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,8 +19,8 @@ jobs: - conda-python-tests - docs-build - devcontainer - - wheel-build-cpp - - wheel-build-python + - wheel-cpp-build + - wheel-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.06 checks: @@ -70,15 +70,15 @@ jobs: sccache -z; build-all --verbose; sccache -s; - wheel-build-cpp: + wheel-cpp-build: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request script: ci/build_wheel_cpp.sh - wheel-build-python: - needs: wheel-build-cpp + wheel-python-build: + needs: wheel-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: diff --git a/build.sh b/build.sh index 769600b10f..6f73dd2002 100755 --- a/build.sh +++ b/build.sh @@ -146,7 +146,7 @@ fi # Build and install the kvikio Python package if (( NUMARGS == 0 )) || hasArg kvikio; then echo "building kvikio..." - cd ${REPODIR}/python + cd ${REPODIR}/python/kvikio SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_LIBRARY_PATH=${LIBKVIKIO_BUILD_DIR};${EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps . fi From f3cfff089331490693382db1bc92788366826f46 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 30 Apr 2024 13:52:22 -0500 Subject: [PATCH 09/76] straighten out paths from python deeper nesting --- .pre-commit-config.yaml | 12 ++++++------ ci/build_wheel_cpp.sh | 4 ++-- ci/build_wheel_python.sh | 4 ++-- dependencies.yaml | 8 +++++--- python/kvikio/pyproject.toml | 16 +++++++--------- python/libkvikio/CMakeLists.txt | 2 +- python/libkvikio/libkvikio/__init__.py | 3 +-- python/libkvikio/libkvikio/_version.py | 1 - 8 files changed, 24 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index be56a05348..a7c0cbf110 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: hooks: - id: black files: python/.* - args: ["--config", "python/pyproject.toml"] + args: ["--config", "python/kvikio/pyproject.toml"] - repo: https://github.com/PyCQA/flake8 rev: 6.0.0 hooks: @@ -37,11 +37,11 @@ repos: hooks: - id: mypy additional_dependencies: [types-cachetools] - args: ["--config-file=python/pyproject.toml", - "python/kvikio", - "python/tests", - "python/examples", - "python/benchmarks"] + args: ["--config-file=python/kvikio/pyproject.toml", + "python/kvikio/kvikio", + "python/kvikio/tests", + "python/kvikio/examples", + "python/kvikio/benchmarks"] pass_filenames: false - repo: https://github.com/pre-commit/mirrors-clang-format rev: v16.0.6 diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 51fcb6d779..b49235922e 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail @@ -35,4 +35,4 @@ python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist \ No newline at end of file +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index ee3f50e502..947959fe83 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. set -euo pipefail @@ -37,4 +37,4 @@ python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find- mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist \ No newline at end of file +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/dependencies.yaml b/dependencies.yaml index 71cbda52ad..c543a9d506 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -52,20 +52,22 @@ files: - depends_on_cupy - run py_wheel_cpp: - output: pyproject/libkvikio + output: pyproject + pyproject_dir: python/kvikio extras: table: project includes: - build py_wheel_python: - output: pyproject/kvikio + output: pyproject + pyproject_dir: python/kvikio extras: table: project includes: - build py_optional_test: output: pyproject - pyproject_dir: python + pyproject_dir: python/kvikio extras: table: project.optional-dependencies key: test diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index 32069c65b8..5bdfdd28d6 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -8,7 +8,7 @@ requires = [ "cython>=3.0.0", "ninja", "scikit-build-core[pyproject]>=0.7.0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] name = "kvikio-cu11" @@ -21,13 +21,11 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cupy-cuda11x>=12.0.0", - "numcodecs <0.12.0", - "numpy>=1.23,<2.0a0", - "libkvikio==24.06.*", - "packaging", - "zarr", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. + "cmake>=3.26.4", + "cython>=3.0.0", + "ninja", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", "Topic :: Database", @@ -45,7 +43,7 @@ test = [ "dask>=2022.05.2", "pytest", "pytest-cov", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project.urls] Homepage = "https://github.com/rapidsai/kvikio" diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 84c53237a2..f260610133 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/libkvikio/libkvikio/__init__.py b/python/libkvikio/libkvikio/__init__.py index 7371ab4c3d..d287d6935e 100644 --- a/python/libkvikio/libkvikio/__init__.py +++ b/python/libkvikio/libkvikio/__init__.py @@ -1,4 +1,3 @@ - # Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from libkvikio._version import __git_commit__, __version__ +from libkvikio._version import __git_commit__, __version__ # noqa: F401 diff --git a/python/libkvikio/libkvikio/_version.py b/python/libkvikio/libkvikio/_version.py index 31548c3a28..b5d848e847 100644 --- a/python/libkvikio/libkvikio/_version.py +++ b/python/libkvikio/libkvikio/_version.py @@ -18,4 +18,3 @@ importlib.resources.files("libkvikio").joinpath("VERSION").read_text().strip() ) __git_commit__ = "" - From 3dceb00bf10e773d6ac6d5848acded2787318ef9 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 30 Apr 2024 14:01:09 -0500 Subject: [PATCH 10/76] linting cmake format --- python/kvikio/CMakeLists.txt | 1 - python/libkvikio/CMakeLists.txt | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index 22436c0839..6107d6d4fd 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -26,7 +26,6 @@ project( LANGUAGES CXX CUDA ) - # TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile include(../cpp/cmake/Modules/FindcuFile.cmake) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index f260610133..99d8ce5465 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -19,7 +19,8 @@ include(../../rapids_config.cmake) project( libkvikio-python VERSION "${RAPIDS_VERSION}" - LANGUAGES CXX) + LANGUAGES CXX +) set(KvikIO_BUILD_EXAMPLES OFF) set(KvikIO_BUILD_TESTS OFF) From 4ebecce75362076b3f90eecc24eaa1264e7c1fe6 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 1 May 2024 16:23:31 -0500 Subject: [PATCH 11/76] add librmm wheel dep from PR, disable cmake find debugging --- ci/build_wheel_cpp.sh | 4 +++- python/libkvikio/CMakeLists.txt | 2 -- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index b49235922e..6f7367c78f 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -28,9 +28,11 @@ if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} fi +librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) + cd "${package_dir}" -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links "${librmm_wheelhouse}" mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 99d8ce5465..f2ffef1ea2 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -26,6 +26,4 @@ set(KvikIO_BUILD_EXAMPLES OFF) set(KvikIO_BUILD_TESTS OFF) set(CUDA_STATIC_RUNTIME ON) -set(CMAKE_FIND_DEBUG_MODE ON) - add_subdirectory(../../cpp kvikio-cpp) From 9206dcd7612c9bd2f03bd82006ce10dc49478ea9 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 1 May 2024 16:26:57 -0500 Subject: [PATCH 12/76] adapt devcontainer run to account for different python package path --- .github/workflows/pr.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5f68b77c53..56c663bd65 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -66,7 +66,12 @@ jobs: with: arch: '["amd64"]' cuda: '["12.2"]' + # HACK: the yq and rapids-generate-script here is a stopgap until both of these PRs are merged: + # - https://github.com/rapidsai/kvikio/pull/369 + # - https://github.com/rapidsai/devcontainers/pull/283 build_command: | + sudo yq -i '.repos[2].python[0].sub_dir = "python/kvikio"' /opt/rapids-build-utils/manifest.yaml; + rapids-generate-scripts; sccache -z; build-all --verbose; sccache -s; From 3298a1a78fd15851d108a0535e1a6aac25f81dd2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 1 May 2024 16:35:02 -0500 Subject: [PATCH 13/76] adjust cpm path; add cpm init to libkvikio wheel --- python/kvikio/CMakeLists.txt | 5 +---- python/libkvikio/CMakeLists.txt | 9 ++++++++- rapids_config.cmake | 36 --------------------------------- 3 files changed, 9 insertions(+), 41 deletions(-) delete mode 100644 rapids_config.cmake diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index 6107d6d4fd..f72f466611 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -14,7 +14,7 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -include(../cpp/cmake/rapids_config.cmake) +include(../../cpp/cmake/rapids_config.cmake) include(rapids-cpm) rapids_cpm_init() include(rapids-cuda) @@ -26,9 +26,6 @@ project( LANGUAGES CXX CUDA ) -# TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile -include(../cpp/cmake/Modules/FindcuFile.cmake) - find_package(KvikIO "${RAPIDS_VERSION}") find_package(CUDAToolkit REQUIRED) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index f2ffef1ea2..b7a377d64a 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -14,7 +14,14 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) -include(../../rapids_config.cmake) +include(../../cpp/cmake/rapids_config.cmake) +include(rapids-cpm) +rapids_cpm_init() +include(rapids-cuda) +rapids_cuda_init_architectures(kvikio-python) + +# TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile +include(../cpp/cmake/Modules/FindcuFile.cmake) project( libkvikio-python diff --git a/rapids_config.cmake b/rapids_config.cmake deleted file mode 100644 index 1413dd28d4..0000000000 --- a/rapids_config.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# ============================================================================= -# Copyright (c) 2018-2024, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= -file(READ "${CMAKE_CURRENT_LIST_DIR}/VERSION" _rapids_version) -if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]]) - set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") - set(RAPIDS_VERSION_MINOR "${CMAKE_MATCH_2}") - set(RAPIDS_VERSION_PATCH "${CMAKE_MATCH_3}") - set(RAPIDS_VERSION_MAJOR_MINOR "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}") - set(RAPIDS_VERSION "${RAPIDS_VERSION_MAJOR}.${RAPIDS_VERSION_MINOR}.${RAPIDS_VERSION_PATCH}") -else() - string(REPLACE "\n" "\n " _rapids_version_formatted " ${_rapids_version}") - message( - FATAL_ERROR - "Could not determine RAPIDS version. Contents of VERSION file:\n${_rapids_version_formatted}" - ) -endif() - -if(NOT EXISTS "${CMAKE_CURRENT_BINARY_DIR}/RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") - file( - DOWNLOAD - "https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION_MAJOR_MINOR}/RAPIDS.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake" - ) -endif() -include("${CMAKE_CURRENT_BINARY_DIR}/RAPIDS-${RAPIDS_VERSION_MAJOR_MINOR}.cmake") From 5002499ba04c9ba12910e97e990f896cbdf3eeb9 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 1 May 2024 16:44:38 -0500 Subject: [PATCH 14/76] fix relpath to findcufile --- python/libkvikio/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index b7a377d64a..d7dfd2db45 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -21,7 +21,7 @@ include(rapids-cuda) rapids_cuda_init_architectures(kvikio-python) # TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile -include(../cpp/cmake/Modules/FindcuFile.cmake) +include(../../cpp/cmake/Modules/FindcuFile.cmake) project( libkvikio-python From 0a9b6505cc36cfda044979bf40f66dba5326830e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 1 May 2024 16:51:50 -0500 Subject: [PATCH 15/76] rapids cuda init architectures fix --- python/libkvikio/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index d7dfd2db45..07ab8b97e0 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -18,7 +18,7 @@ include(../../cpp/cmake/rapids_config.cmake) include(rapids-cpm) rapids_cpm_init() include(rapids-cuda) -rapids_cuda_init_architectures(kvikio-python) +rapids_cuda_init_architectures(libkvikio-python) # TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile include(../../cpp/cmake/Modules/FindcuFile.cmake) From eebdb557c3f844e98d01bfe687e9fdf865d31b41 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Fri, 3 May 2024 09:02:33 -0500 Subject: [PATCH 16/76] Apply suggestions from code review Co-authored-by: Vyas Ramasubramani --- dependencies.yaml | 2 +- python/kvikio/CMakeLists.txt | 2 +- python/kvikio/kvikio/__init__.py | 11 ----------- python/kvikio/kvikio/_version.py | 2 +- 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index c543a9d506..235f431937 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -53,7 +53,7 @@ files: - run py_wheel_cpp: output: pyproject - pyproject_dir: python/kvikio + pyproject_dir: python/libkvikio extras: table: project includes: diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index f72f466611..34122276b8 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -26,7 +26,7 @@ project( LANGUAGES CXX CUDA ) -find_package(KvikIO "${RAPIDS_VERSION}") +find_package(KvikIO "${RAPIDS_VERSION}" REQUIRED) find_package(CUDAToolkit REQUIRED) diff --git a/python/kvikio/kvikio/__init__.py b/python/kvikio/kvikio/__init__.py index a280a370c3..5186df50f8 100644 --- a/python/kvikio/kvikio/__init__.py +++ b/python/kvikio/kvikio/__init__.py @@ -1,17 +1,6 @@ # Copyright (c) 2021-2024, NVIDIA CORPORATION. All rights reserved. # See file LICENSE for terms. - -# If libcudf was installed as a wheel, we must request it to load the library symbols. -# Otherwise, we assume that the library was installed in a system path that ld can find. -try: - import libkvikio -except ModuleNotFoundError: - pass -else: - libkvikio.load_library() - del libkvikio - from ._lib import libkvikio # type: ignore from ._version import __version__ # noqa: F401 from .cufile import CuFile # noqa: F401 diff --git a/python/kvikio/kvikio/_version.py b/python/kvikio/kvikio/_version.py index 75cad4e207..82fc478d5c 100644 --- a/python/kvikio/kvikio/_version.py +++ b/python/kvikio/kvikio/_version.py @@ -17,4 +17,4 @@ __version__ = ( importlib.resources.files("kvikio").joinpath("VERSION").read_text().strip() ) -__git_commit__ = "7b0231c8241164c8a0511cd737e7098cb7ccea3e" +__git_commit__ = "" From 7f2a9d7ec7a4455f73e0fe70611ebead98f40d25 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 09:03:21 -0500 Subject: [PATCH 17/76] adjusting dependencies and wheel tagging --- .github/workflows/pr.yaml | 7 +++++++ VERSION | 2 +- ci/build_wheel_cpp.sh | 6 ++---- dependencies.yaml | 1 + python/kvikio/.gitattributes | 1 - python/kvikio/pyproject.toml | 1 + python/libkvikio/.gitattributes | 1 - python/libkvikio/pyproject.toml | 20 +++++--------------- 8 files changed, 17 insertions(+), 22 deletions(-) delete mode 100644 python/kvikio/.gitattributes delete mode 100644 python/libkvikio/.gitattributes diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 56c663bd65..3782cf51c5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -89,3 +89,10 @@ jobs: with: build_type: pull-request script: ci/build_wheel_python.sh + wheel-python-tests: + needs: wheel-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 + with: + build_type: pull-request + script: ci/run_pytests.sh diff --git a/VERSION b/VERSION index cb16728b39..0bff6981a3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.06.00a11 +24.06.00 diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 6f7367c78f..0ebf4a389b 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -33,8 +33,6 @@ librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-p cd "${package_dir}" python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links "${librmm_wheelhouse}" +python -m wheel tags --platform any dist/* --remove -mkdir -p final_dist -python -m auditwheel repair -w final_dist dist/* - -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist diff --git a/dependencies.yaml b/dependencies.yaml index 235f431937..73268bcc42 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -65,6 +65,7 @@ files: table: project includes: - build + - wheel_lib_dep py_optional_test: output: pyproject pyproject_dir: python/kvikio diff --git a/python/kvikio/.gitattributes b/python/kvikio/.gitattributes deleted file mode 100644 index 2656ae48af..0000000000 --- a/python/kvikio/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -kvikio/_version.py export-subst diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index 5bdfdd28d6..e94f3abe72 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -23,6 +23,7 @@ requires-python = ">=3.9" dependencies = [ "cmake>=3.26.4", "cython>=3.0.0", + "libkvikio==24.6.*", "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/libkvikio/.gitattributes b/python/libkvikio/.gitattributes deleted file mode 100644 index a2d7dee9e0..0000000000 --- a/python/libkvikio/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -libkvikio/_version.py export-subst diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index 897996409a..872aee6e7f 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -19,14 +19,12 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -requires-python = ">=3.9" dependencies = [ - "cupy-cuda11x>=12.0.0", - "numcodecs <0.12.0", - "numpy>=1.23,<2.0a0", - "packaging", - "zarr", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. + "cmake>=3.26.4", + "cython>=3.0.0", + "ninja", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", "Topic :: Database", @@ -38,14 +36,6 @@ classifiers = [ "Programming Language :: Python :: 3.11", ] -[project.optional-dependencies] -test = [ - "cuda-python>=11.7.1,<12.0a0", - "dask>=2022.05.2", - "pytest", - "pytest-cov", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. - [project.urls] Homepage = "https://github.com/rapidsai/kvikio" From e0166518aed49190df5eb3da43a82b514bcae641 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 09:10:48 -0500 Subject: [PATCH 18/76] remove build deps from py_wheel_cpp --- python/libkvikio/pyproject.toml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index 872aee6e7f..77ea9eb9eb 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -3,12 +3,7 @@ [build-system] build-backend = "scikit_build_core.build" -requires = [ - "cmake>=3.26.4", - "cython>=3.0.0", - "ninja", - "scikit-build-core[pyproject]>=0.7.0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`. +requires = [] [project] name = "libkvikio" From e056546cfe57dd21a66779e21ed04f43c222b9ba Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 09:12:27 -0500 Subject: [PATCH 19/76] remove findCufile in libkvikio wheel cmakelists --- python/libkvikio/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 07ab8b97e0..24fb7aab7e 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -20,9 +20,6 @@ rapids_cpm_init() include(rapids-cuda) rapids_cuda_init_architectures(libkvikio-python) -# TODO: Should we symlink FindcuFile.cmake into python/cmake? find cuFile -include(../../cpp/cmake/Modules/FindcuFile.cmake) - project( libkvikio-python VERSION "${RAPIDS_VERSION}" From 55349a01fdb9c0b751b31ef8120510757ff4b423 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 09:14:41 -0500 Subject: [PATCH 20/76] add wheel-python-tests in pr.yaml --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3782cf51c5..69c46edbb0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -21,6 +21,7 @@ jobs: - devcontainer - wheel-cpp-build - wheel-python-build + - wheel-python-tests secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.06 checks: From 3bc31e26c23d04d4807a08800c6e4e14fadc5c5b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 09:31:08 -0500 Subject: [PATCH 21/76] comment librmm inclusion --- ci/build_wheel_cpp.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 0ebf4a389b..4864c2f8dc 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -28,11 +28,11 @@ if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} fi -librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) +# librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) cd "${package_dir}" -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links "${librmm_wheelhouse}" +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # --find-links "${librmm_wheelhouse}" python -m wheel tags --platform any dist/* --remove RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist From 1249409476db35e9b9c2f95003c8530c3b46c8ca Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 10:38:45 -0500 Subject: [PATCH 22/76] fix dependency on scikit-build for wheel --- ci/run_pytests.sh | 2 +- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - conda/environments/all_cuda-122_arch-x86_64.yaml | 1 - dependencies.yaml | 14 ++++++++++---- python/kvikio/pyproject.toml | 1 - 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/ci/run_pytests.sh b/ci/run_pytests.sh index 73de68d03b..b2c93dbe56 100755 --- a/ci/run_pytests.sh +++ b/ci/run_pytests.sh @@ -4,6 +4,6 @@ set -euo pipefail # Support invoking run_pytests.sh outside the script directory -cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/ +cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/kvikio pytest --cache-clear --verbose "$@" tests diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 03c84b29a7..0bf956e999 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -30,7 +30,6 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 -- scikit-build-core>=0.7.0 - sphinx - sphinx-click - sphinx_rtd_theme diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index e1a8f31c5b..438d894af3 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -28,7 +28,6 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 -- scikit-build-core>=0.7.0 - sphinx - sphinx-click - sphinx_rtd_theme diff --git a/dependencies.yaml b/dependencies.yaml index 73268bcc42..6f1da55a23 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -58,6 +58,7 @@ files: table: project includes: - build + - scikit-build py_wheel_python: output: pyproject pyproject_dir: python/kvikio @@ -65,6 +66,7 @@ files: table: project includes: - build + - scikit-build - wheel_lib_dep py_optional_test: output: pyproject @@ -91,10 +93,6 @@ dependencies: packages: - c-compiler - cxx-compiler - - scikit-build-core>=0.7.0 - - output_types: [requirements, pyproject] - packages: - - scikit-build-core[pyproject]>=0.7.0 specific: - output_types: conda matrices: @@ -124,6 +122,14 @@ dependencies: cuda: "12.*" packages: - cuda-nvcc + scikit-build: + common: + - output_types: conda + packages: + - scikit-build-core>=0.7.0 + - output_types: [requirements, pyproject] + packages: + - scikit-build-core[pyproject]>=0.7.0 checks: common: - output_types: [conda, requirements] diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index e94f3abe72..ae8a7049a2 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -7,7 +7,6 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.0", "ninja", - "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] From 430ffd2940dd926b4301ab70721d03b39e935c3c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 12:20:43 -0500 Subject: [PATCH 23/76] fix dependency on scikit-build for wheel --- .../all_cuda-118_arch-x86_64.yaml | 1 + .../all_cuda-122_arch-x86_64.yaml | 1 + dependencies.yaml | 32 +++++++++++++------ python/kvikio/pyproject.toml | 1 + python/libkvikio/pyproject.toml | 1 - 5 files changed, 26 insertions(+), 10 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 0bf956e999..03c84b29a7 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -30,6 +30,7 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 +- scikit-build-core>=0.7.0 - sphinx - sphinx-click - sphinx_rtd_theme diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index 438d894af3..e1a8f31c5b 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -28,6 +28,7 @@ dependencies: - pytest - pytest-cov - python>=3.9,<3.12 +- scikit-build-core>=0.7.0 - sphinx - sphinx-click - sphinx_rtd_theme diff --git a/dependencies.yaml b/dependencies.yaml index 6f1da55a23..31ea7bb18e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -6,7 +6,10 @@ files: cuda: ["11.8", "12.2"] arch: [x86_64] includes: - - build + - build-universal + - build-cpp + - build-py + - build-py-wrapper - checks - cuda - cuda_version @@ -42,7 +45,10 @@ files: extras: table: build-system includes: - - build + - build-universal + - build-cpp + - build-py + - build-py-wrapper py_run: output: pyproject pyproject_dir: python/kvikio @@ -57,16 +63,18 @@ files: extras: table: project includes: - - build - - scikit-build + - build-universal + - build-py py_wheel_python: output: pyproject pyproject_dir: python/kvikio extras: table: project includes: - - build - - scikit-build + - build-universal + - build-cpp + - build-py + - build-py-wrapper - wheel_lib_dep py_optional_test: output: pyproject @@ -82,13 +90,14 @@ channels: - conda-forge - nvidia dependencies: - build: + build-universal: common: - output_types: [conda, requirements, pyproject] packages: - cmake>=3.26.4 - - cython>=3.0.0 - ninja + build-cpp: + common: - output_types: conda packages: - c-compiler @@ -122,7 +131,7 @@ dependencies: cuda: "12.*" packages: - cuda-nvcc - scikit-build: + build-py: common: - output_types: conda packages: @@ -130,6 +139,11 @@ dependencies: - output_types: [requirements, pyproject] packages: - scikit-build-core[pyproject]>=0.7.0 + build-py-wrapper: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cython>=3.0.0 checks: common: - output_types: [conda, requirements] diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index ae8a7049a2..e94f3abe72 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -7,6 +7,7 @@ requires = [ "cmake>=3.26.4", "cython>=3.0.0", "ninja", + "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index 77ea9eb9eb..42c394a6f6 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -16,7 +16,6 @@ authors = [ license = { text = "Apache 2.0" } dependencies = [ "cmake>=3.26.4", - "cython>=3.0.0", "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 977d9059aa14007265c4be812d3f45b4a7fab8ed Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 12:38:13 -0500 Subject: [PATCH 24/76] fix dependency on scikit-build for wheel --- dependencies.yaml | 13 +++++-------- python/kvikio/pyproject.toml | 9 +++++---- python/libkvikio/pyproject.toml | 6 +++++- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 31ea7bb18e..c9de0d2e5b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -61,7 +61,7 @@ files: output: pyproject pyproject_dir: python/libkvikio extras: - table: project + table: build-system includes: - build-universal - build-py @@ -69,13 +69,12 @@ files: output: pyproject pyproject_dir: python/kvikio extras: - table: project + table: build-system includes: - build-universal - build-cpp - build-py - build-py-wrapper - - wheel_lib_dep py_optional_test: output: pyproject pyproject_dir: python/kvikio @@ -243,11 +242,6 @@ dependencies: packages: &cupy_packages_cu11 - cupy-cuda11x>=12.0.0 - {matrix: null, packages: *cupy_packages_cu11} - wheel_lib_dep: - common: - - output_types: pyproject - packages: - - libkvikio==24.6.* docs: common: - output_types: [conda, requirements] @@ -287,6 +281,9 @@ dependencies: # See https://github.com/zarr-developers/numcodecs/pull/475 - numcodecs <0.12.0 - packaging + - output_types: [requirements, pyproject] + packages: + - libkvikio==24.6.* test_python: common: - output_types: [conda, requirements, pyproject] diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index e94f3abe72..fc734c353b 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -21,11 +21,12 @@ authors = [ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ - "cmake>=3.26.4", - "cython>=3.0.0", + "cupy-cuda11x>=12.0.0", "libkvikio==24.6.*", - "ninja", - "scikit-build-core[pyproject]>=0.7.0", + "numcodecs <0.12.0", + "numpy>=1.23,<2.0a0", + "packaging", + "zarr", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index 42c394a6f6..f2b1560a63 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -3,7 +3,11 @@ [build-system] build-backend = "scikit_build_core.build" -requires = [] +requires = [ + "cmake>=3.26.4", + "ninja", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] name = "libkvikio" From 28f0dbd5ec6afef88cdae0ba48d1b69805c08eea Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 12:43:27 -0500 Subject: [PATCH 25/76] add wheel dep --- conda/environments/all_cuda-118_arch-x86_64.yaml | 1 + conda/environments/all_cuda-122_arch-x86_64.yaml | 1 + dependencies.yaml | 2 ++ python/kvikio/pyproject.toml | 1 + python/libkvikio/pyproject.toml | 1 + 5 files changed, 6 insertions(+) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index 03c84b29a7..b912c27521 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -35,5 +35,6 @@ dependencies: - sphinx-click - sphinx_rtd_theme - sysroot_linux-64=2.17 +- wheel - zarr name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index e1a8f31c5b..e9e18c1655 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -33,5 +33,6 @@ dependencies: - sphinx-click - sphinx_rtd_theme - sysroot_linux-64=2.17 +- wheel - zarr name: all_cuda-122_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index c9de0d2e5b..f759524537 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -135,9 +135,11 @@ dependencies: - output_types: conda packages: - scikit-build-core>=0.7.0 + - wheel - output_types: [requirements, pyproject] packages: - scikit-build-core[pyproject]>=0.7.0 + - wheel build-py-wrapper: common: - output_types: [conda, requirements, pyproject] diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index fc734c353b..3a000fe349 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -8,6 +8,7 @@ requires = [ "cython>=3.0.0", "ninja", "scikit-build-core[pyproject]>=0.7.0", + "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index f2b1560a63..d010f2e4a3 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -7,6 +7,7 @@ requires = [ "cmake>=3.26.4", "ninja", "scikit-build-core[pyproject]>=0.7.0", + "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] From 4fd111f3d71a2181d90fdbb051e6ec3e345a20da Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 13:39:26 -0500 Subject: [PATCH 26/76] wheel has to be installed in build scripts --- ci/build_wheel_cpp.sh | 1 + ci/build_wheel_python.sh | 1 + conda/environments/all_cuda-118_arch-x86_64.yaml | 1 - conda/environments/all_cuda-122_arch-x86_64.yaml | 1 - dependencies.yaml | 4 +--- docs/source/zarr.rst | 2 +- python/kvikio/pyproject.toml | 1 - python/libkvikio/pyproject.toml | 1 - 8 files changed, 4 insertions(+), 8 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 4864c2f8dc..6ddb906801 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -32,6 +32,7 @@ fi cd "${package_dir}" +python -m pip install -y wheel python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # --find-links "${librmm_wheelhouse}" python -m wheel tags --platform any dist/* --remove diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 947959fe83..290ebf3a50 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -32,6 +32,7 @@ CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download cd "${package_dir}" +python -m pip install -y wheel python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE} mkdir -p final_dist diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index b912c27521..03c84b29a7 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -35,6 +35,5 @@ dependencies: - sphinx-click - sphinx_rtd_theme - sysroot_linux-64=2.17 -- wheel - zarr name: all_cuda-118_arch-x86_64 diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml index e9e18c1655..e1a8f31c5b 100644 --- a/conda/environments/all_cuda-122_arch-x86_64.yaml +++ b/conda/environments/all_cuda-122_arch-x86_64.yaml @@ -33,6 +33,5 @@ dependencies: - sphinx-click - sphinx_rtd_theme - sysroot_linux-64=2.17 -- wheel - zarr name: all_cuda-122_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index f759524537..f4646418d3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -135,11 +135,9 @@ dependencies: - output_types: conda packages: - scikit-build-core>=0.7.0 - - wheel - output_types: [requirements, pyproject] packages: - scikit-build-core[pyproject]>=0.7.0 - - wheel build-py-wrapper: common: - output_types: [conda, requirements, pyproject] @@ -283,7 +281,7 @@ dependencies: # See https://github.com/zarr-developers/numcodecs/pull/475 - numcodecs <0.12.0 - packaging - - output_types: [requirements, pyproject] + - output_types: [pyproject] packages: - libkvikio==24.6.* test_python: diff --git a/docs/source/zarr.rst b/docs/source/zarr.rst index f2a697d525..5b63ffd8b7 100644 --- a/docs/source/zarr.rst +++ b/docs/source/zarr.rst @@ -11,5 +11,5 @@ The following is an example of how to use the convenience function :py:meth:`kvi to create a new Zarr array and how open an existing Zarr array. -.. literalinclude:: ../../python/examples/zarr_cupy_nvcomp.py +.. literalinclude:: ../../python/kvikio/examples/zarr_cupy_nvcomp.py :language: python diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index 3a000fe349..fc734c353b 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -8,7 +8,6 @@ requires = [ "cython>=3.0.0", "ninja", "scikit-build-core[pyproject]>=0.7.0", - "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index d010f2e4a3..f2b1560a63 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -7,7 +7,6 @@ requires = [ "cmake>=3.26.4", "ninja", "scikit-build-core[pyproject]>=0.7.0", - "wheel", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] From e2e73b810ee3d46a1b2b8570b7408628cd96937a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 14:35:27 -0500 Subject: [PATCH 27/76] wheel has to be installed in build scripts --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 6ddb906801..78fcea3288 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -32,7 +32,7 @@ fi cd "${package_dir}" -python -m pip install -y wheel +python -m pip install wheel python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # --find-links "${librmm_wheelhouse}" python -m wheel tags --platform any dist/* --remove diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 290ebf3a50..dd40118f8a 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -32,7 +32,7 @@ CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download cd "${package_dir}" -python -m pip install -y wheel +python -m pip install wheel python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE} mkdir -p final_dist From 550b21d43081a98d344e6be2eb865c41de4bb067 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 15:23:17 -0500 Subject: [PATCH 28/76] fix s3 download cpp wheel filename --- ci/build_wheel_python.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index dd40118f8a..4852e8c39a 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -28,7 +28,7 @@ if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} fi -CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) cd "${package_dir}" From 71ba1409f81e7eaebb05726c3807701f8de9f234 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 May 2024 17:46:46 -0500 Subject: [PATCH 29/76] move libkvikio wheel to build reqs for kvikio wheel (header-only!) --- dependencies.yaml | 9 ++++++--- python/kvikio/pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index f4646418d3..2125bbd6ec 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -75,6 +75,7 @@ files: - build-cpp - build-py - build-py-wrapper + - build-use-libkvikio-wheel py_optional_test: output: pyproject pyproject_dir: python/kvikio @@ -138,6 +139,11 @@ dependencies: - output_types: [requirements, pyproject] packages: - scikit-build-core[pyproject]>=0.7.0 + build-use-libkvikio-wheel: + common: + - output_types: [pyproject] + packages: + - libkvikio==24.6.* build-py-wrapper: common: - output_types: [conda, requirements, pyproject] @@ -281,9 +287,6 @@ dependencies: # See https://github.com/zarr-developers/numcodecs/pull/475 - numcodecs <0.12.0 - packaging - - output_types: [pyproject] - packages: - - libkvikio==24.6.* test_python: common: - output_types: [conda, requirements, pyproject] diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index fc734c353b..ea502e024b 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -6,6 +6,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.0", + "libkvikio==24.6.*", "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -22,7 +23,6 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cupy-cuda11x>=12.0.0", - "libkvikio==24.6.*", "numcodecs <0.12.0", "numpy>=1.23,<2.0a0", "packaging", From 33d5b8a86d5298db36c0630cd7542d94fab4ff5e Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Mon, 6 May 2024 11:10:41 -0500 Subject: [PATCH 30/76] add wheel.py-api to libkvikio wheel --- python/libkvikio/pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index f2b1560a63..f2084bb1d8 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -102,6 +102,7 @@ ninja.make-fallback = true sdist.exclude = ["*tests*"] sdist.reproducible = true wheel.packages = ["libkvikio"] +wheel.py-api = "py3" [tool.scikit-build.metadata.version] provider = "scikit_build_core.metadata.regex" From 04801ef7f1b497d49f075550d1c35b99696de76d Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 6 May 2024 11:53:20 -0500 Subject: [PATCH 31/76] fix missing -cuXY version for libkvikio --- ci/build_wheel_python.sh | 6 ++++-- python/kvikio/pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 4852e8c39a..5dc02b0802 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -24,9 +24,11 @@ sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFF echo "${version}" > VERSION sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' fi +sed -r -i "s/libkvikio==(.*)\"/libkvikio${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index ea502e024b..b23136a4e7 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -12,7 +12,7 @@ requires = [ ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [project] -name = "kvikio-cu11" +name = "kvikio" dynamic = ["version"] description = "KvikIO - GPUDirect Storage" readme = { file = "README.md", content-type = "text/markdown" } From ff4e2bb6a37d0e7122742dbaebf272c990d7481a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 07:55:16 -0500 Subject: [PATCH 32/76] add install_aliased_imported_targets for nvcomp --- python/kvikio/CMakeLists.txt | 7 ++ .../kvikio/cmake/Modules/WheelHelpers.cmake | 71 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 python/kvikio/cmake/Modules/WheelHelpers.cmake diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index 34122276b8..c9a1dda1a5 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -30,6 +30,13 @@ find_package(KvikIO "${RAPIDS_VERSION}" REQUIRED) find_package(CUDAToolkit REQUIRED) +# It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it here. +include(cmake/Modules/WheelHelpers.cmake) +install_aliased_imported_targets( + TARGETS kvikio nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp + DESTINATION ${cython_lib_dir} +) + if(NOT KvikIO_FOUND) add_subdirectory(../cpp kvikio-cpp) set(cython_lib_dir kvikio) diff --git a/python/kvikio/cmake/Modules/WheelHelpers.cmake b/python/kvikio/cmake/Modules/WheelHelpers.cmake new file mode 100644 index 0000000000..c0351e8bbc --- /dev/null +++ b/python/kvikio/cmake/Modules/WheelHelpers.cmake @@ -0,0 +1,71 @@ +# ============================================================================= +# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= +include_guard(GLOBAL) + +# Making libraries available inside wheels by installing the associated targets. +function(add_target_libs_to_wheel) + list(APPEND CMAKE_MESSAGE_CONTEXT "add_target_libs_to_wheel") + + set(options "") + set(one_value "LIB_DIR") + set(multi_value "TARGETS") + cmake_parse_arguments(_ "${options}" "${one_value}" "${multi_value}" ${ARGN}) + + message(VERBOSE "Installing targets '${__TARGETS}' into lib_dir '${__LIB_DIR}'") + + foreach(target IN LISTS __TARGETS) + + if(NOT TARGET ${target}) + message(VERBOSE "No target named ${target}") + continue() + endif() + + get_target_property(alias_target ${target} ALIASED_TARGET) + if(alias_target) + set(target ${alias_target}) + endif() + + get_target_property(is_imported ${target} IMPORTED) + if(NOT is_imported) + # If the target isn't imported, install it into the wheel + install(TARGETS ${target} DESTINATION ${__LIB_DIR}) + message(VERBOSE "install(TARGETS ${target} DESTINATION ${__LIB_DIR})") + else() + # If the target is imported, make sure it's global + get_target_property(already_global ${target} IMPORTED_GLOBAL) + if(NOT already_global) + set_target_properties(${target} PROPERTIES IMPORTED_GLOBAL TRUE) + endif() + + # Find the imported target's library so we can copy it into the wheel + set(lib_loc) + foreach(prop IN ITEMS IMPORTED_LOCATION IMPORTED_LOCATION_RELEASE IMPORTED_LOCATION_DEBUG) + get_target_property(lib_loc ${target} ${prop}) + if(lib_loc) + message(VERBOSE "Found ${prop} for ${target}: ${lib_loc}") + break() + endif() + message(VERBOSE "${target} has no value for property ${prop}") + endforeach() + + if(NOT lib_loc) + message(FATAL_ERROR "Found no libs to install for target ${target}") + endif() + + # Copy the imported library into the wheel + install(FILES ${lib_loc} DESTINATION ${__LIB_DIR}) + message(VERBOSE "install(FILES ${lib_loc} DESTINATION ${__LIB_DIR})") + endif() + endforeach() +endfunction() From abef47d7a2a489f4630858a472f31a9f796f4074 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 08:06:11 -0500 Subject: [PATCH 33/76] style --- python/kvikio/CMakeLists.txt | 15 ++++++++------- python/kvikio/cmake/Modules/WheelHelpers.cmake | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index c9a1dda1a5..bd5b5cef85 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -30,13 +30,6 @@ find_package(KvikIO "${RAPIDS_VERSION}" REQUIRED) find_package(CUDAToolkit REQUIRED) -# It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it here. -include(cmake/Modules/WheelHelpers.cmake) -install_aliased_imported_targets( - TARGETS kvikio nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp - DESTINATION ${cython_lib_dir} -) - if(NOT KvikIO_FOUND) add_subdirectory(../cpp kvikio-cpp) set(cython_lib_dir kvikio) @@ -46,5 +39,13 @@ endif() include(rapids-cython-core) rapids_cython_init() +# It would be better to factor nvcomp out into its own wheel. Until that is available, +# we vendor it here. +include(cmake/Modules/WheelHelpers.cmake) +install_aliased_imported_targets( + TARGETS kvikio nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION + ${cython_lib_dir} +) + add_subdirectory(cmake) add_subdirectory(kvikio/_lib) diff --git a/python/kvikio/cmake/Modules/WheelHelpers.cmake b/python/kvikio/cmake/Modules/WheelHelpers.cmake index c0351e8bbc..2a68fda311 100644 --- a/python/kvikio/cmake/Modules/WheelHelpers.cmake +++ b/python/kvikio/cmake/Modules/WheelHelpers.cmake @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at From 4331fd8ed18b5796f59285784a01c87a9f864276 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 08:24:16 -0500 Subject: [PATCH 34/76] try to get CMake install_aliased_imported_targets working --- python/kvikio/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index bd5b5cef85..6bdd8ea656 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -39,11 +39,11 @@ endif() include(rapids-cython-core) rapids_cython_init() -# It would be better to factor nvcomp out into its own wheel. Until that is available, -# we vendor it here. +# It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it +# here. include(cmake/Modules/WheelHelpers.cmake) install_aliased_imported_targets( - TARGETS kvikio nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION + TARGETS kvikio-python nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION ${cython_lib_dir} ) From 899f2497e916d992f1d17e49b16bb01a73b8c675 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 09:49:44 -0500 Subject: [PATCH 35/76] remove unused get_nvcomp.cmake --- python/kvikio/CMakeLists.txt | 5 +-- python/kvikio/cmake/CMakeLists.txt | 15 --------- .../kvikio/cmake/thirdparty/get_nvcomp.cmake | 33 ------------------- 3 files changed, 3 insertions(+), 50 deletions(-) delete mode 100644 python/kvikio/cmake/CMakeLists.txt delete mode 100644 python/kvikio/cmake/thirdparty/get_nvcomp.cmake diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index 6bdd8ea656..3a9356d416 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -18,7 +18,6 @@ include(../../cpp/cmake/rapids_config.cmake) include(rapids-cpm) rapids_cpm_init() include(rapids-cuda) -rapids_cuda_init_architectures(kvikio-python) project( kvikio-python @@ -26,12 +25,14 @@ project( LANGUAGES CXX CUDA ) +rapids_cuda_init_architectures(kvikio-python) + find_package(KvikIO "${RAPIDS_VERSION}" REQUIRED) find_package(CUDAToolkit REQUIRED) if(NOT KvikIO_FOUND) - add_subdirectory(../cpp kvikio-cpp) + add_subdirectory(../../cpp kvikio-cpp) set(cython_lib_dir kvikio) install(TARGETS kvikio DESTINATION ${cython_lib_dir}) endif() diff --git a/python/kvikio/cmake/CMakeLists.txt b/python/kvikio/cmake/CMakeLists.txt deleted file mode 100644 index 4249773480..0000000000 --- a/python/kvikio/cmake/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -include(thirdparty/get_nvcomp.cmake) diff --git a/python/kvikio/cmake/thirdparty/get_nvcomp.cmake b/python/kvikio/cmake/thirdparty/get_nvcomp.cmake deleted file mode 100644 index ff39366bff..0000000000 --- a/python/kvikio/cmake/thirdparty/get_nvcomp.cmake +++ /dev/null @@ -1,33 +0,0 @@ -# ============================================================================= -# Copyright (c) 2021-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except -# in compliance with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under -# the License. -# ============================================================================= - -set(KVIKIO_USE_PROPRIETARY_BINARY ON) - -# This function finds nvcomp and sets any additional necessary environment variables. -function(find_and_configure_nvcomp) - - include(${rapids-cmake-dir}/cpm/nvcomp.cmake) - rapids_cpm_nvcomp( - BUILD_EXPORT_SET kvikio-exports - INSTALL_EXPORT_SET kvikio-exports - USE_PROPRIETARY_BINARY ${KVIKIO_USE_PROPRIETARY_BINARY} - ) - - # Per-thread default stream - if(TARGET nvcomp AND PER_THREAD_DEFAULT_STREAM) - target_compile_definitions(nvcomp PRIVATE CUDA_API_PER_THREAD_DEFAULT_STREAM) - endif() -endfunction() - -find_and_configure_nvcomp() From ab4deb592bcd0fda2fdcdfb478f55c441dc8a296 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 10:42:33 -0500 Subject: [PATCH 36/76] fix wrong WheelHelpers.cmake --- .../kvikio/cmake/Modules/WheelHelpers.cmake | 44 +++++++------------ 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/python/kvikio/cmake/Modules/WheelHelpers.cmake b/python/kvikio/cmake/Modules/WheelHelpers.cmake index 2a68fda311..3abe98a064 100644 --- a/python/kvikio/cmake/Modules/WheelHelpers.cmake +++ b/python/kvikio/cmake/Modules/WheelHelpers.cmake @@ -14,15 +14,15 @@ include_guard(GLOBAL) # Making libraries available inside wheels by installing the associated targets. -function(add_target_libs_to_wheel) - list(APPEND CMAKE_MESSAGE_CONTEXT "add_target_libs_to_wheel") +function(install_aliased_imported_targets) + list(APPEND CMAKE_MESSAGE_CONTEXT "install_aliased_imported_targets") set(options "") - set(one_value "LIB_DIR") + set(one_value "DESTINATION") set(multi_value "TARGETS") cmake_parse_arguments(_ "${options}" "${one_value}" "${multi_value}" ${ARGN}) - message(VERBOSE "Installing targets '${__TARGETS}' into lib_dir '${__LIB_DIR}'") + message(VERBOSE "Installing targets '${__TARGETS}' into lib_dir '${__DESTINATION}'") foreach(target IN LISTS __TARGETS) @@ -39,33 +39,21 @@ function(add_target_libs_to_wheel) get_target_property(is_imported ${target} IMPORTED) if(NOT is_imported) # If the target isn't imported, install it into the wheel - install(TARGETS ${target} DESTINATION ${__LIB_DIR}) - message(VERBOSE "install(TARGETS ${target} DESTINATION ${__LIB_DIR})") + install(TARGETS ${target} DESTINATION ${__DESTINATION}) + message(VERBOSE "install(TARGETS ${target} DESTINATION ${__DESTINATION})") else() # If the target is imported, make sure it's global - get_target_property(already_global ${target} IMPORTED_GLOBAL) - if(NOT already_global) - set_target_properties(${target} PROPERTIES IMPORTED_GLOBAL TRUE) + get_target_property(type ${target} TYPE) + if(${type} STREQUAL "UNKNOWN_LIBRARY") + install(FILES $ DESTINATION ${__DESTINATION}) + message(VERBOSE "install(FILES $ DESTINATION ${__DESTINATION})") + else() + install(IMPORTED_RUNTIME_ARTIFACTS ${target} DESTINATION ${__DESTINATION}) + message( + VERBOSE + "install(IMPORTED_RUNTIME_ARTIFACTS $ DESTINATION ${__DESTINATION})" + ) endif() - - # Find the imported target's library so we can copy it into the wheel - set(lib_loc) - foreach(prop IN ITEMS IMPORTED_LOCATION IMPORTED_LOCATION_RELEASE IMPORTED_LOCATION_DEBUG) - get_target_property(lib_loc ${target} ${prop}) - if(lib_loc) - message(VERBOSE "Found ${prop} for ${target}: ${lib_loc}") - break() - endif() - message(VERBOSE "${target} has no value for property ${prop}") - endforeach() - - if(NOT lib_loc) - message(FATAL_ERROR "Found no libs to install for target ${target}") - endif() - - # Copy the imported library into the wheel - install(FILES ${lib_loc} DESTINATION ${__LIB_DIR}) - message(VERBOSE "install(FILES ${lib_loc} DESTINATION ${__LIB_DIR})") endif() endforeach() endfunction() From a048172df031821e21a018695e6215b4a5595f6c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 10:55:07 -0500 Subject: [PATCH 37/76] reorganize python kvikio cmake --- python/kvikio/CMakeLists.txt | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index 3a9356d416..044e58a9ff 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -18,6 +18,7 @@ include(../../cpp/cmake/rapids_config.cmake) include(rapids-cpm) rapids_cpm_init() include(rapids-cuda) +rapids_cuda_init_architectures(kvikio-python) project( kvikio-python @@ -25,28 +26,18 @@ project( LANGUAGES CXX CUDA ) -rapids_cuda_init_architectures(kvikio-python) - find_package(KvikIO "${RAPIDS_VERSION}" REQUIRED) - find_package(CUDAToolkit REQUIRED) - -if(NOT KvikIO_FOUND) - add_subdirectory(../../cpp kvikio-cpp) - set(cython_lib_dir kvikio) - install(TARGETS kvikio DESTINATION ${cython_lib_dir}) -endif() +add_subdirectory(cmake/Modules) include(rapids-cython-core) rapids_cython_init() # It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it # here. -include(cmake/Modules/WheelHelpers.cmake) install_aliased_imported_targets( TARGETS kvikio-python nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION ${cython_lib_dir} ) -add_subdirectory(cmake) add_subdirectory(kvikio/_lib) From a58f44584090f5d8d8261b901f334bd2dc398cd1 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 11:04:22 -0500 Subject: [PATCH 38/76] restore get_nvcomp.cmake, move WheelHelpers.cmake --- python/kvikio/CMakeLists.txt | 3 +- python/kvikio/cmake/CMakeLists.txt | 16 +++++++++ .../WheelHelpers.cmake | 0 .../kvikio/cmake/thirdparty/get_nvcomp.cmake | 33 +++++++++++++++++++ 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 python/kvikio/cmake/CMakeLists.txt rename python/kvikio/cmake/{Modules => thirdparty}/WheelHelpers.cmake (100%) create mode 100644 python/kvikio/cmake/thirdparty/get_nvcomp.cmake diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index 044e58a9ff..dd751bb9aa 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -28,11 +28,12 @@ project( find_package(KvikIO "${RAPIDS_VERSION}" REQUIRED) find_package(CUDAToolkit REQUIRED) -add_subdirectory(cmake/Modules) include(rapids-cython-core) rapids_cython_init() +add_subdirectory(cmake) + # It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it # here. install_aliased_imported_targets( diff --git a/python/kvikio/cmake/CMakeLists.txt b/python/kvikio/cmake/CMakeLists.txt new file mode 100644 index 0000000000..051b52731e --- /dev/null +++ b/python/kvikio/cmake/CMakeLists.txt @@ -0,0 +1,16 @@ +# ============================================================================= +# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +include(thirdparty/get_nvcomp.cmake) +include(thirdpart/WheelHelpers.cmake) diff --git a/python/kvikio/cmake/Modules/WheelHelpers.cmake b/python/kvikio/cmake/thirdparty/WheelHelpers.cmake similarity index 100% rename from python/kvikio/cmake/Modules/WheelHelpers.cmake rename to python/kvikio/cmake/thirdparty/WheelHelpers.cmake diff --git a/python/kvikio/cmake/thirdparty/get_nvcomp.cmake b/python/kvikio/cmake/thirdparty/get_nvcomp.cmake new file mode 100644 index 0000000000..ff39366bff --- /dev/null +++ b/python/kvikio/cmake/thirdparty/get_nvcomp.cmake @@ -0,0 +1,33 @@ +# ============================================================================= +# Copyright (c) 2021-2023, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +set(KVIKIO_USE_PROPRIETARY_BINARY ON) + +# This function finds nvcomp and sets any additional necessary environment variables. +function(find_and_configure_nvcomp) + + include(${rapids-cmake-dir}/cpm/nvcomp.cmake) + rapids_cpm_nvcomp( + BUILD_EXPORT_SET kvikio-exports + INSTALL_EXPORT_SET kvikio-exports + USE_PROPRIETARY_BINARY ${KVIKIO_USE_PROPRIETARY_BINARY} + ) + + # Per-thread default stream + if(TARGET nvcomp AND PER_THREAD_DEFAULT_STREAM) + target_compile_definitions(nvcomp PRIVATE CUDA_API_PER_THREAD_DEFAULT_STREAM) + endif() +endfunction() + +find_and_configure_nvcomp() From e8d1f223da9fea6ecc84649ea46683a57065ba5a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 11:09:32 -0500 Subject: [PATCH 39/76] typo in cmake thirdparty include --- python/kvikio/cmake/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/kvikio/cmake/CMakeLists.txt b/python/kvikio/cmake/CMakeLists.txt index 051b52731e..0ea444a905 100644 --- a/python/kvikio/cmake/CMakeLists.txt +++ b/python/kvikio/cmake/CMakeLists.txt @@ -13,4 +13,4 @@ # ============================================================================= include(thirdparty/get_nvcomp.cmake) -include(thirdpart/WheelHelpers.cmake) +include(thirdparty/WheelHelpers.cmake) From 891c1ce191922ebd930647a82f2b1bc8ef1e885f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 11:35:58 -0500 Subject: [PATCH 40/76] set cython lib dir --- python/kvikio/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index dd751bb9aa..f891f56d40 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -29,6 +29,7 @@ project( find_package(KvikIO "${RAPIDS_VERSION}" REQUIRED) find_package(CUDAToolkit REQUIRED) +set(cython_lib_dir kvikio) include(rapids-cython-core) rapids_cython_init() From a8d7377810136bdfda6785c84199cc2bd1c15e1c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 11:48:05 -0500 Subject: [PATCH 41/76] install nvcomp stuff into _lib subfolder alongside cython .so's --- python/kvikio/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index f891f56d40..c22a3da652 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -38,8 +38,8 @@ add_subdirectory(cmake) # It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it # here. install_aliased_imported_targets( - TARGETS kvikio-python nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION - ${cython_lib_dir} + TARGETS kvikio nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION + ${cython_lib_dir}/_lib ) add_subdirectory(kvikio/_lib) From f4fbf11fd9bf7e38f28b639038ed8ee72863610f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 14:01:11 -0500 Subject: [PATCH 42/76] add simple wheel install/test script --- .github/workflows/pr.yaml | 2 +- ci/test_wheel.sh | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 ci/test_wheel.sh diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 69c46edbb0..1d59afaa11 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -96,4 +96,4 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request - script: ci/run_pytests.sh + script: ci/test_wheel.sh diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh new file mode 100644 index 0000000000..955a3a7645 --- /dev/null +++ b/ci/test_wheel.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -eou pipefail + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" + +# echo to expand wildcard before adding `[extra]` requires for pip +python -m pip install "kvikio-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" + +python -m pytest ./python/kvikio/kvikio/tests From a33f3fa6f793fb78ad6ab0225741381cf11bbd00 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 14:17:51 -0500 Subject: [PATCH 43/76] chmod +x test_wheel.sh --- ci/test_wheel.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ci/test_wheel.sh diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh old mode 100644 new mode 100755 From efa116e12f5e9bef1edb0b65f50e5a5d71f8c03f Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 14:37:56 -0500 Subject: [PATCH 44/76] fix nesting too deep in test folder --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 955a3a7645..fde2f88000 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -10,4 +10,4 @@ RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-fr # echo to expand wildcard before adding `[extra]` requires for pip python -m pip install "kvikio-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" -python -m pytest ./python/kvikio/kvikio/tests +python -m pytest ./python/kvikio/tests From 5eb6b801bc68130d9e689aec1579a276d00c8ebf Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Tue, 7 May 2024 17:00:17 -0500 Subject: [PATCH 45/76] Apply suggestions from code review Co-authored-by: Vyas Ramasubramani --- .github/workflows/pr.yaml | 2 -- ci/build_wheel_cpp.sh | 4 +-- ci/test_wheel.sh | 1 - python/kvikio/CMakeLists.txt | 2 +- python/libkvikio/CMakeLists.txt | 2 +- python/libkvikio/pyproject.toml | 62 --------------------------------- 6 files changed, 3 insertions(+), 70 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1d59afaa11..913f235728 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -71,8 +71,6 @@ jobs: # - https://github.com/rapidsai/kvikio/pull/369 # - https://github.com/rapidsai/devcontainers/pull/283 build_command: | - sudo yq -i '.repos[2].python[0].sub_dir = "python/kvikio"' /opt/rapids-build-utils/manifest.yaml; - rapids-generate-scripts; sccache -z; build-all --verbose; sccache -s; diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 78fcea3288..1703165d74 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -28,12 +28,10 @@ if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} fi -# librmm_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="${RAPIDS_PY_CUDA_SUFFIX}" rapids-get-pr-wheel-artifact rmm 1529 cpp) - cd "${package_dir}" python -m pip install wheel -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check # --find-links "${librmm_wheelhouse}" +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check python -m wheel tags --platform any dist/* --remove RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index fde2f88000..5e4c622c7d 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -7,7 +7,6 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" WHEELHOUSE="${PWD}/dist/" RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" -# echo to expand wildcard before adding `[extra]` requires for pip python -m pip install "kvikio-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" python -m pytest ./python/kvikio/tests diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index c22a3da652..a727ed23bf 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -38,7 +38,7 @@ add_subdirectory(cmake) # It would be better to factor nvcomp out into its own wheel. Until that is available, we vendor it # here. install_aliased_imported_targets( - TARGETS kvikio nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION + TARGETS nvcomp::nvcomp nvcomp::nvcomp_gdeflate nvcomp::nvcomp_bitcomp DESTINATION ${cython_lib_dir}/_lib ) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 24fb7aab7e..b966064aeb 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index f2084bb1d8..d48256bdba 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -18,11 +18,6 @@ authors = [ { name = "NVIDIA Corporation" }, ] license = { text = "Apache 2.0" } -dependencies = [ - "cmake>=3.26.4", - "ninja", - "scikit-build-core[pyproject]>=0.7.0", -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. classifiers = [ "Intended Audience :: Developers", "Topic :: Database", @@ -37,63 +32,6 @@ classifiers = [ [project.urls] Homepage = "https://github.com/rapidsai/kvikio" -[tool.black] -line-length = 88 -target-version = ["py39"] -include = '\.py?$' -exclude = ''' -/( - thirdparty | - \.eggs | - \.git | - \.hg | - \.mypy_cache | - \.tox | - \.venv | - _build | - buck-out | - build | - dist | - _skbuild -)/ -''' - -[tool.isort] -line_length = 88 -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -combine_as_imports = true -order_by_type = true -known_first_party = [ - "kvikio", -] -default_section = "THIRDPARTY" -sections = [ - "FUTURE", - "STDLIB", - "THIRDPARTY", - "FIRSTPARTY", - "LOCALFOLDER", -] -skip = [ - "thirdparty", - ".eggs", - ".git", - ".hg", - ".mypy_cache", - ".tox", - ".venv", - "_build", - "buck-out", - "build", - "dist", - "__init__.py", -] - -[tool.mypy] -ignore_missing_imports = true - [tool.scikit-build] build-dir = "build/{wheel_tag}" cmake.build-type = "Release" From 798c60b08812c0ab973b0212019c2a0c484e8617 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 17:28:37 -0500 Subject: [PATCH 46/76] fix cuda-python not adjusting for cuda version --- ci/build_wheel_python.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 5dc02b0802..0ec32a750e 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -30,6 +30,10 @@ if ! rapids-is-release-build; then fi sed -r -i "s/libkvikio==(.*)\"/libkvikio${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} +if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} +fi + CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) cd "${package_dir}" From de87c1ccbd2725e7ee948a498484a2b1973355b3 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 17:29:07 -0500 Subject: [PATCH 47/76] adjust more copyright dates --- python/kvikio/pyproject.toml | 2 +- python/libkvikio/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index 5640d82723..c3a86f1299 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # See file LICENSE for terms. [build-system] diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index d48256bdba..a995dce26b 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -1,4 +1,4 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. # See file LICENSE for terms. [build-system] From 2b22c5a29ccdd6d3689116dbe81cbb2a92cd94b0 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 7 May 2024 18:21:50 -0500 Subject: [PATCH 48/76] add sed for cupy-cuda11x -> cupy-cuda12x on cuda 12 --- ci/build_wheel_python.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 0ec32a750e..fb6fd678c3 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -32,6 +32,7 @@ sed -r -i "s/libkvikio==(.*)\"/libkvikio${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\ if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} fi CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) From 3d97ce392a48a56b4ab6cb91ac508a504b74b29c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 8 May 2024 11:18:10 -0500 Subject: [PATCH 49/76] add early cmake exit in libkvikio wheel if kvikio is already available --- python/libkvikio/CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index b966064aeb..0cf8676e1a 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -26,6 +26,16 @@ project( LANGUAGES CXX ) +# Check if kvikio is already available. If so, it's the user's responsibility to ensure that the CMake +# package is also available at build time of the Python kvikio package. +find_package(kvikio "${RAPIDS_VERSION}") + +if(kvikio_FOUND) + return() +endif() + +unset(kvikio_FOUND) + set(KvikIO_BUILD_EXAMPLES OFF) set(KvikIO_BUILD_TESTS OFF) set(CUDA_STATIC_RUNTIME ON) From 0860a9920b8751a419489caa58ecbe6725c8279c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 8 May 2024 11:18:32 -0500 Subject: [PATCH 50/76] add wheel publishing in build.yaml --- .github/workflows/build.yaml | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 4e6cca8727..de11d6a266 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,3 +66,45 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} + wheel-build-cpp: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_cpp.sh + wheel-build-python: + needs: wheel-build-cpp + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_python.sh + wheel-publish-cpp: + needs: wheel-build-cpp + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: kvikio + package-type: cpp + wheel-publish-python: + needs: wheel-build-python + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: kvikio + package-type: python From 9b89b291b1adcedb5e41bda1ab87f4434fcdbc58 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 8 May 2024 11:20:59 -0500 Subject: [PATCH 51/76] cmake style fix --- python/libkvikio/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 0cf8676e1a..830cc5ea6f 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -26,8 +26,8 @@ project( LANGUAGES CXX ) -# Check if kvikio is already available. If so, it's the user's responsibility to ensure that the CMake -# package is also available at build time of the Python kvikio package. +# Check if kvikio is already available. If so, it's the user's responsibility to ensure that the +# CMake package is also available at build time of the Python kvikio package. find_package(kvikio "${RAPIDS_VERSION}") if(kvikio_FOUND) From 0e4beeed5d9f677b18f7b74732b46d4b66ce8908 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Wed, 8 May 2024 12:24:07 -0500 Subject: [PATCH 52/76] Update .github/workflows/build.yaml Co-authored-by: Vyas Ramasubramani --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index de11d6a266..ecdd5b4b90 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -95,7 +95,7 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: kvikio + package-name: libkvikio package-type: cpp wheel-publish-python: needs: wheel-build-python From d82635e1869673861c6904369c8e710815b0deff Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 5 Jun 2024 23:19:48 -0700 Subject: [PATCH 53/76] Bump libkvikio version --- dependencies.yaml | 2 +- python/kvikio/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 2125bbd6ec..ff85090955 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -143,7 +143,7 @@ dependencies: common: - output_types: [pyproject] packages: - - libkvikio==24.6.* + - libkvikio==24.8.* build-py-wrapper: common: - output_types: [conda, requirements, pyproject] diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index c3a86f1299..49d7f3251f 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.0", - "libkvikio==24.6.*", + "libkvikio==24.8.*", "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From bbed671a6b1b0e5b8838c39d3959876c08964acc Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 5 Jun 2024 23:28:51 -0700 Subject: [PATCH 54/76] Use latest workflows --- .github/workflows/build.yaml | 8 ++++---- .github/workflows/pr.yaml | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8b5874a179..7b826f0cb4 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -68,7 +68,7 @@ jobs: sha: ${{ inputs.sha }} wheel-build-cpp: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} @@ -79,7 +79,7 @@ jobs: wheel-build-python: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -89,7 +89,7 @@ jobs: wheel-publish-cpp: needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -100,7 +100,7 @@ jobs: wheel-publish-python: needs: wheel-build-python secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a2e07c6f16..a4712a8aa2 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -76,7 +76,7 @@ jobs: sccache -s; wheel-cpp-build: secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request @@ -84,14 +84,14 @@ jobs: wheel-python-build: needs: wheel-cpp-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: build_type: pull-request script: ci/build_wheel_python.sh wheel-python-tests: needs: wheel-python-build secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: build_type: pull-request script: ci/test_wheel.sh From 3dce0766d942efd3de74793273f9fdab9a9db0f4 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 5 Jun 2024 23:38:33 -0700 Subject: [PATCH 55/76] Address reviewers' comments --- ci/build_wheel_cpp.sh | 4 ---- cpp/CMakeLists.txt | 14 ++++++++++++-- python/libkvikio/pyproject.toml | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 1703165d74..70deecbe7a 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -24,10 +24,6 @@ sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFF echo "${version}" > VERSION sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} -fi - cd "${package_dir}" python -m pip install wheel diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b1ac294d7a..b9382e447b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -95,7 +95,13 @@ add_library(kvikio::kvikio ALIAS kvikio) # Enable CUDA in KvikIO if(CUDAToolkit_FOUND) - target_link_libraries(kvikio INTERFACE $) + if(CUDA_STATIC_RUNTIME) + target_link_libraries(kvikio INTERFACE $ + $) + else() + target_link_libraries(kvikio INTERFACE $ + $) + endif() target_compile_definitions(kvikio INTERFACE $) else() message(WARNING "Building KvikIO without CUDA") @@ -167,7 +173,11 @@ if(NOT already_set_kvikio) find_package(CUDAToolkit QUIET) if(CUDAToolkit_FOUND) - target_link_libraries(kvikio::kvikio INTERFACE CUDA::toolkit) + if(CUDA_STATIC_RUNTIME) + target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart_static CUDA::cuFile_static) + else() + target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart CUDA::cuFile) + endif() target_compile_definitions(kvikio::kvikio INTERFACE KVIKIO_CUDA_FOUND) else() message(WARNING "Building KvikIO without CUDA") diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index a995dce26b..7418a31614 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -12,7 +12,7 @@ requires = [ [project] name = "libkvikio" dynamic = ["version"] -description = "KvikIO - GPUDirect Storage" +description = "KvikIO - GPUDirect Storage (C++)" readme = { file = "README.md", content-type = "text/markdown" } authors = [ { name = "NVIDIA Corporation" }, From ff2182ac0c5d538498333f659e79db091e38336b Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 5 Jun 2024 23:48:43 -0700 Subject: [PATCH 56/76] Fix formatting --- cpp/CMakeLists.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b9382e447b..ef040a412b 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -96,11 +96,14 @@ add_library(kvikio::kvikio ALIAS kvikio) # Enable CUDA in KvikIO if(CUDAToolkit_FOUND) if(CUDA_STATIC_RUNTIME) - target_link_libraries(kvikio INTERFACE $ - $) + target_link_libraries( + kvikio INTERFACE $ + $ + ) else() - target_link_libraries(kvikio INTERFACE $ - $) + target_link_libraries( + kvikio INTERFACE $ $ + ) endif() target_compile_definitions(kvikio INTERFACE $) else() From 1b415f0d7791e7bcddecf5767d67cd11c86b596d Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 6 Jun 2024 06:32:52 -0700 Subject: [PATCH 57/76] Defer to existing FindcuFile module --- cpp/CMakeLists.txt | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index ef040a412b..956a5cfcee 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -96,14 +96,9 @@ add_library(kvikio::kvikio ALIAS kvikio) # Enable CUDA in KvikIO if(CUDAToolkit_FOUND) if(CUDA_STATIC_RUNTIME) - target_link_libraries( - kvikio INTERFACE $ - $ - ) + target_link_libraries(kvikio INTERFACE $) else() - target_link_libraries( - kvikio INTERFACE $ $ - ) + target_link_libraries(kvikio INTERFACE $) endif() target_compile_definitions(kvikio INTERFACE $) else() @@ -177,9 +172,9 @@ if(NOT already_set_kvikio) find_package(CUDAToolkit QUIET) if(CUDAToolkit_FOUND) if(CUDA_STATIC_RUNTIME) - target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart_static CUDA::cuFile_static) + target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart_static) else() - target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart CUDA::cuFile) + target_link_libraries(kvikio::kvikio INTERFACE CUDA::cudart) endif() target_compile_definitions(kvikio::kvikio INTERFACE KVIKIO_CUDA_FOUND) else() From 3d1ac243812c31470b9aced790191588d251e645 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Thu, 6 Jun 2024 15:26:57 -0700 Subject: [PATCH 58/76] Remove unneeded build.sh --- conda/recipes/kvikio/build.sh | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 conda/recipes/kvikio/build.sh diff --git a/conda/recipes/kvikio/build.sh b/conda/recipes/kvikio/build.sh deleted file mode 100644 index 462f06826b..0000000000 --- a/conda/recipes/kvikio/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -# Copyright (c) 2018-2024, NVIDIA CORPORATION. - -./build.sh kvikio From 8f058b9f6c8b5ae1ecfd36d045a99ce8e7a883f7 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Fri, 7 Jun 2024 14:10:53 -0700 Subject: [PATCH 59/76] Apply suggestions from code review Co-authored-by: James Lamb --- ci/test_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheel.sh b/ci/test_wheel.sh index 5e4c622c7d..94a31b04b6 100755 --- a/ci/test_wheel.sh +++ b/ci/test_wheel.sh @@ -7,6 +7,6 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" WHEELHOUSE="${PWD}/dist/" RAPIDS_PY_WHEEL_NAME="kvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" -python -m pip install "kvikio-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" +python -m pip install "$(echo ${WHEELHOUSE}/kvikio_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" python -m pytest ./python/kvikio/tests From ccbfd2223e99a1cc1d832ca48c19929f43d8330a Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Fri, 7 Jun 2024 14:10:08 -0700 Subject: [PATCH 60/76] Removed outdated comment --- .github/workflows/pr.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a4712a8aa2..3f84139578 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -67,9 +67,6 @@ jobs: with: arch: '["amd64"]' cuda: '["12.2"]' - # HACK: the yq and rapids-generate-script here is a stopgap until both of these PRs are merged: - # - https://github.com/rapidsai/kvikio/pull/369 - # - https://github.com/rapidsai/devcontainers/pull/283 build_command: | sccache -z; build-all --verbose; From 84d9e330e4ce4d1e7c93f8ff6b05b845f1342b1c Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Fri, 7 Jun 2024 14:13:07 -0700 Subject: [PATCH 61/76] Use a separate build.sh --- conda/recipes/kvikio/build.sh | 4 ++++ conda/recipes/kvikio/meta.yaml | 3 --- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 conda/recipes/kvikio/build.sh diff --git a/conda/recipes/kvikio/build.sh b/conda/recipes/kvikio/build.sh new file mode 100644 index 0000000000..462f06826b --- /dev/null +++ b/conda/recipes/kvikio/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +# Copyright (c) 2018-2024, NVIDIA CORPORATION. + +./build.sh kvikio diff --git a/conda/recipes/kvikio/meta.yaml b/conda/recipes/kvikio/meta.yaml index adb3ccaeb4..886334be5d 100644 --- a/conda/recipes/kvikio/meta.yaml +++ b/conda/recipes/kvikio/meta.yaml @@ -32,9 +32,6 @@ build: - SCCACHE_S3_KEY_PREFIX=kvikio-linux64 # [linux64] - SCCACHE_S3_USE_SSL - SCCACHE_S3_NO_CREDENTIALS - script: - - cd python/kvikio - - python -m pip install . -vv ignore_run_exports_from: {% if cuda_major == "11" %} - {{ compiler('cuda11') }} From 5b9da4a6e16dccb3e406a2e168d0dd1f590b6967 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Jun 2024 12:10:36 -0500 Subject: [PATCH 62/76] rapids-build-backend for python extension-only wheel --- dependencies.yaml | 16 +++------------- python/kvikio/pyproject.toml | 6 ++---- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index b985807f91..819c7e03ab 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -47,18 +47,6 @@ files: table: build-system includes: - rapids_build_skbuild - py_rapids_build: - output: pyproject - pyproject_dir: python/kvikio - extras: - table: tool.rapids-build-backend - key: requires - includes: - - build-universal - - build-cpp - - build-py - - build-py-wrapper - - rapids_build_skbuild py_run: output: pyproject pyproject_dir: python/kvikio @@ -79,13 +67,15 @@ files: output: pyproject pyproject_dir: python/kvikio extras: - table: build-system + table: tool.rapids-build-backend + key: requires includes: - build-universal - build-cpp - build-py - build-py-wrapper - build-use-libkvikio-wheel + - rapids_build_skbuild py_optional_test: output: pyproject pyproject_dir: python/kvikio diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index c46bd762e7..ded3f1bae7 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -4,10 +4,7 @@ [build-system] build-backend = "rapids_build_backend.build" requires = [ - "cmake>=3.26.4", - "cython>=3.0.0", - "libkvikio==24.8.*", - "ninja", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -116,6 +113,7 @@ dependencies-file = "../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.0", + "libkvikio==24.8.*", "ninja", "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", From 7c83f47311a8e33092cf7b02a39cede437718950 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Jun 2024 12:26:08 -0500 Subject: [PATCH 63/76] fix dependencies-file relative path --- python/kvikio/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index ded3f1bae7..a5c898b27c 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -109,7 +109,7 @@ nvcomp_batch = "kvikio.nvcomp_codec:NvCompBatchCodec" [tool.rapids-build-backend] build-backend = "scikit_build_core.build" -dependencies-file = "../dependencies.yaml" +dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.0", From e2e46fee026905ce1bbcf53061974020dd17ca62 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Jun 2024 13:16:13 -0500 Subject: [PATCH 64/76] always set cython_lib_dir --- python/kvikio/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/kvikio/CMakeLists.txt b/python/kvikio/CMakeLists.txt index 191fd0658d..9217645637 100644 --- a/python/kvikio/CMakeLists.txt +++ b/python/kvikio/CMakeLists.txt @@ -41,9 +41,10 @@ endif() find_package(CUDAToolkit REQUIRED) +set(cython_lib_dir kvikio) + if(NOT KvikIO_FOUND) add_subdirectory(../../cpp kvikio-cpp) - set(cython_lib_dir kvikio) install(TARGETS kvikio DESTINATION ${cython_lib_dir}) endif() From cbfd5f5bdb534258376bc4c3c18a6e4e7d7ec4e5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Jun 2024 14:25:32 -0500 Subject: [PATCH 65/76] attempt to cut out sed stuff that RBB obviates --- ci/build_wheel_cpp.sh | 7 +------ ci/build_wheel_python.sh | 15 +-------------- 2 files changed, 2 insertions(+), 20 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 70deecbe7a..a29ba10b81 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -9,8 +9,7 @@ package_dir="python/libkvikio" source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) +rapids-generate-version > ./VERSION RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" @@ -20,10 +19,6 @@ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" pyproject_file="${package_dir}/pyproject.toml" -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" - cd "${package_dir}" python -m pip install wheel diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index fb6fd678c3..4c419a7289 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -20,20 +20,7 @@ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" pyproject_file="${package_dir}/pyproject.toml" -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py" - -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi -sed -r -i "s/libkvikio==(.*)\"/libkvikio${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi +rapids-generate-version > ./VERSION CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) From a32ad58c1d1628788efa3d3acb171ff552d079c0 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 17 Jun 2024 15:47:08 -0500 Subject: [PATCH 66/76] add RBB for libkvikio --- dependencies.yaml | 4 +++- python/libkvikio/pyproject.toml | 10 +++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 819c7e03ab..94069e03ec 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -59,10 +59,12 @@ files: output: pyproject pyproject_dir: python/libkvikio extras: - table: build-system + table: tool.rapids-build-backend + key: requires includes: - build-universal - build-py + - rapids_build_skbuild py_wheel_python: output: pyproject pyproject_dir: python/kvikio diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index 7418a31614..df5d13ce52 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -2,7 +2,7 @@ # See file LICENSE for terms. [build-system] -build-backend = "scikit_build_core.build" +build-backend = "rapids_build_backend.build" requires = [ "cmake>=3.26.4", "ninja", @@ -46,3 +46,11 @@ wheel.py-api = "py3" provider = "scikit_build_core.metadata.regex" input = "libkvikio/VERSION" regex = "(?P.*)" + +[tool.rapids-build-backend] +requires = [ + "cmake>=3.26.4", + "ninja", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From df85d255ab1697bf79f87cb08db0e9ffbe4fba47 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Jun 2024 09:16:02 -0500 Subject: [PATCH 67/76] install RBB in build_wheel_*scripts --- ci/build_wheel_cpp.sh | 2 +- ci/build_wheel_python.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index a29ba10b81..fc59541753 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -21,7 +21,7 @@ pyproject_file="${package_dir}/pyproject.toml" cd "${package_dir}" -python -m pip install wheel +python -m pip install wheel rapids-build-backend python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check python -m wheel tags --platform any dist/* --remove diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 4c419a7289..42aae15862 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -26,7 +26,7 @@ CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapid cd "${package_dir}" -python -m pip install wheel +python -m pip install wheel rapids-build-backend python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE} mkdir -p final_dist From db0ce3bb75ddded1302ed3bdc2a1275bdbb7b707 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Jun 2024 09:25:39 -0500 Subject: [PATCH 68/76] rbb in build deps for libkvikio --- dependencies.yaml | 7 +++++++ python/libkvikio/pyproject.toml | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 94069e03ec..8f1e289efd 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -47,6 +47,13 @@ files: table: build-system includes: - rapids_build_skbuild + py_build_cpp_wheel: + output: pyproject + pyproject_dir: python/libkvikio + extras: + table: build-system + includes: + - rapids_build_skbuild py_run: output: pyproject pyproject_dir: python/kvikio diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index df5d13ce52..f82ca54297 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -4,8 +4,7 @@ [build-system] build-backend = "rapids_build_backend.build" requires = [ - "cmake>=3.26.4", - "ninja", + "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From bf7dc543e3313564f26564a629967e81a9fe9614 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Jun 2024 09:44:11 -0500 Subject: [PATCH 69/76] tweak build-backend in libkvikio pyproject.toml --- dependencies.yaml | 2 -- python/kvikio/pyproject.toml | 1 - python/libkvikio/pyproject.toml | 3 ++- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 8f1e289efd..7bdc4c315d 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -71,7 +71,6 @@ files: includes: - build-universal - build-py - - rapids_build_skbuild py_wheel_python: output: pyproject pyproject_dir: python/kvikio @@ -84,7 +83,6 @@ files: - build-py - build-py-wrapper - build-use-libkvikio-wheel - - rapids_build_skbuild py_optional_test: output: pyproject pyproject_dir: python/kvikio diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index a5c898b27c..b54e1afaa8 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -115,7 +115,6 @@ requires = [ "cython>=3.0.0", "libkvikio==24.8.*", "ninja", - "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index f82ca54297..5ba8af6fd2 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -47,9 +47,10 @@ input = "libkvikio/VERSION" regex = "(?P.*)" [tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "ninja", - "rapids-build-backend>=0.3.0,<0.4.0.dev0", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From f461ab2132e95f1524b0ed2aa229285e979a35a0 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Jun 2024 11:13:35 -0500 Subject: [PATCH 70/76] libkvikio dynamic dep for pyproject --- dependencies.yaml | 12 +++++++++++- python/kvikio/pyproject.toml | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 7bdc4c315d..2713d5f1b4 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -148,9 +148,19 @@ dependencies: - scikit-build-core[pyproject]>=0.7.0 build-use-libkvikio-wheel: common: - - output_types: [pyproject] + - output_types: conda packages: - libkvikio==24.8.* + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - libkvikio-cu12==24.8.* + - matrix: {cuda: "11.*"} + packages: &libkvikio_packages_cu11 + - libkvikio-cu11==24.8.* + - {matrix: null, packages: *libkvikio_packages_cu11} build-py-wrapper: common: - output_types: [conda, requirements, pyproject] diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index b54e1afaa8..524e8c4793 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -113,7 +113,7 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.0", - "libkvikio==24.8.*", + "libkvikio-cu11==24.8.*", "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 5e2560b0565f1464d8464cf391816d8f6d6f7055 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Jun 2024 11:48:41 -0500 Subject: [PATCH 71/76] allow alpha specs --- dependencies.yaml | 6 +++--- python/kvikio/pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 2713d5f1b4..61b1e4dfb8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -150,16 +150,16 @@ dependencies: common: - output_types: conda packages: - - libkvikio==24.8.* + - libkvikio==24.8.*,>=0.0.0a0 specific: - output_types: [requirements, pyproject] matrices: - matrix: {cuda: "12.*"} packages: - - libkvikio-cu12==24.8.* + - libkvikio-cu12==24.8.*,>=0.0.0a0 - matrix: {cuda: "11.*"} packages: &libkvikio_packages_cu11 - - libkvikio-cu11==24.8.* + - libkvikio-cu11==24.8.*,>=0.0.0a0 - {matrix: null, packages: *libkvikio_packages_cu11} build-py-wrapper: common: diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index 524e8c4793..c01a47c196 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -113,7 +113,7 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.0", - "libkvikio-cu11==24.8.*", + "libkvikio-cu11==24.8.*,>=0.0.0a0", "ninja", "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From dd2d246bebfc332016b8fb6a78f7a0ae703f390e Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Jun 2024 12:21:48 -0500 Subject: [PATCH 72/76] clean up some unnecessary changes --- ci/build_wheel_cpp.sh | 9 ++------- ci/build_wheel_python.sh | 6 +----- python/kvikio/cmake/CMakeLists.txt | 3 +-- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index fc59541753..ecbba13c8b 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -11,18 +11,13 @@ source rapids-date-string rapids-generate-version > ./VERSION -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - pyproject_file="${package_dir}/pyproject.toml" cd "${package_dir}" -python -m pip install wheel rapids-build-backend +python -m pip install wheel python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check python -m wheel tags --platform any dist/* --remove +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp dist diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 42aae15862..78bf458803 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -14,10 +14,6 @@ commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - pyproject_file="${package_dir}/pyproject.toml" rapids-generate-version > ./VERSION @@ -26,7 +22,7 @@ CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapid cd "${package_dir}" -python -m pip install wheel rapids-build-backend +python -m pip install wheel python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE} mkdir -p final_dist diff --git a/python/kvikio/cmake/CMakeLists.txt b/python/kvikio/cmake/CMakeLists.txt index 0ea444a905..4249773480 100644 --- a/python/kvikio/cmake/CMakeLists.txt +++ b/python/kvikio/cmake/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2023, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -13,4 +13,3 @@ # ============================================================================= include(thirdparty/get_nvcomp.cmake) -include(thirdparty/WheelHelpers.cmake) From 5706ee32d4263b3e4d202c647f15844ec5f6a253 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Jun 2024 12:27:33 -0500 Subject: [PATCH 73/76] revert WheelHelpers removal and add note about install_aliased_imported_targets --- python/kvikio/cmake/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/kvikio/cmake/CMakeLists.txt b/python/kvikio/cmake/CMakeLists.txt index 4249773480..fa94bc3f8e 100644 --- a/python/kvikio/cmake/CMakeLists.txt +++ b/python/kvikio/cmake/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -13,3 +13,5 @@ # ============================================================================= include(thirdparty/get_nvcomp.cmake) +# Needed for install_aliased_imported_targets +include(thirdparty/WheelHelpers.cmake) From b728c93b92b6e024f1cf95391de73c935d576df3 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Tue, 18 Jun 2024 14:31:58 -0500 Subject: [PATCH 74/76] Apply suggestions from code review Co-authored-by: James Lamb --- ci/build_wheel_cpp.sh | 1 - ci/build_wheel_python.sh | 3 --- 2 files changed, 4 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index ecbba13c8b..f6b4f2cab7 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -11,7 +11,6 @@ source rapids-date-string rapids-generate-version > ./VERSION -pyproject_file="${package_dir}/pyproject.toml" cd "${package_dir}" diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 78bf458803..1e6e4efac4 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -9,12 +9,9 @@ package_dir="python/kvikio" source rapids-configure-sccache source rapids-date-string -version=$(rapids-generate-version) -commit=$(git rev-parse HEAD) RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -pyproject_file="${package_dir}/pyproject.toml" rapids-generate-version > ./VERSION From 5a8ab667f0b00f9d8f7e5f6c214c610718f54e0a Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 18 Jun 2024 14:35:09 -0500 Subject: [PATCH 75/76] clean up extraneous skbuild stuff (from review) --- ci/build_wheel_cpp.sh | 1 - dependencies.yaml | 17 +++-------------- python/kvikio/pyproject.toml | 3 +-- python/libkvikio/libkvikio/__init__.py | 4 +++- python/libkvikio/libkvikio/_version.py | 14 ++++++++++++-- python/libkvikio/pyproject.toml | 1 - 6 files changed, 19 insertions(+), 21 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index f6b4f2cab7..2bcd58ae5e 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -11,7 +11,6 @@ source rapids-date-string rapids-generate-version > ./VERSION - cd "${package_dir}" python -m pip install wheel diff --git a/dependencies.yaml b/dependencies.yaml index 61b1e4dfb8..0ed8cae952 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -8,7 +8,6 @@ files: includes: - build-universal - build-cpp - - build-py - build-py-wrapper - checks - cuda @@ -70,7 +69,6 @@ files: key: requires includes: - build-universal - - build-py py_wheel_python: output: pyproject pyproject_dir: python/kvikio @@ -80,7 +78,6 @@ files: includes: - build-universal - build-cpp - - build-py - build-py-wrapper - build-use-libkvikio-wheel py_optional_test: @@ -138,18 +135,10 @@ dependencies: cuda: "12.*" packages: - cuda-nvcc - build-py: - common: - - output_types: conda - packages: - - scikit-build-core>=0.7.0 - - output_types: [requirements, pyproject] - packages: - - scikit-build-core[pyproject]>=0.7.0 build-use-libkvikio-wheel: common: - output_types: conda - packages: + packages: &libkvikio_packages - libkvikio==24.8.*,>=0.0.0a0 specific: - output_types: [requirements, pyproject] @@ -158,9 +147,9 @@ dependencies: packages: - libkvikio-cu12==24.8.*,>=0.0.0a0 - matrix: {cuda: "11.*"} - packages: &libkvikio_packages_cu11 + packages: - libkvikio-cu11==24.8.*,>=0.0.0a0 - - {matrix: null, packages: *libkvikio_packages_cu11} + - {matrix: null, packages: *libkvikio_packages} build-py-wrapper: common: - output_types: [conda, requirements, pyproject] diff --git a/python/kvikio/pyproject.toml b/python/kvikio/pyproject.toml index c01a47c196..f93685f2b3 100644 --- a/python/kvikio/pyproject.toml +++ b/python/kvikio/pyproject.toml @@ -113,9 +113,8 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "cython>=3.0.0", - "libkvikio-cu11==24.8.*,>=0.0.0a0", + "libkvikio==24.8.*,>=0.0.0a0", "ninja", - "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.scikit-build] diff --git a/python/libkvikio/libkvikio/__init__.py b/python/libkvikio/libkvikio/__init__.py index d287d6935e..ed1426bd9a 100644 --- a/python/libkvikio/libkvikio/__init__.py +++ b/python/libkvikio/libkvikio/__init__.py @@ -12,4 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from libkvikio._version import __git_commit__, __version__ # noqa: F401 +from libkvikio._version import __git_commit__, __version__ + +__all__ = [__git_commit__, __version__] diff --git a/python/libkvikio/libkvikio/_version.py b/python/libkvikio/libkvikio/_version.py index b5d848e847..8b67326806 100644 --- a/python/libkvikio/libkvikio/_version.py +++ b/python/libkvikio/libkvikio/_version.py @@ -15,6 +15,16 @@ import importlib.resources __version__ = ( - importlib.resources.files("libkvikio").joinpath("VERSION").read_text().strip() + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() ) -__git_commit__ = "" +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__version__", "__git_commit__"] diff --git a/python/libkvikio/pyproject.toml b/python/libkvikio/pyproject.toml index 5ba8af6fd2..628e44f1b8 100644 --- a/python/libkvikio/pyproject.toml +++ b/python/libkvikio/pyproject.toml @@ -52,5 +52,4 @@ dependencies-file = "../../dependencies.yaml" requires = [ "cmake>=3.26.4", "ninja", - "scikit-build-core[pyproject]>=0.7.0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 3cf7f823f7e7bdf1a4bdaad3ad6f4d2a3d310855 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 18 Jun 2024 15:59:05 -0700 Subject: [PATCH 76/76] Apply suggestions from code review --- ci/build_wheel_cpp.sh | 4 ++++ ci/build_wheel_python.sh | 3 --- python/libkvikio/CMakeLists.txt | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh index 2bcd58ae5e..9893474da3 100755 --- a/ci/build_wheel_cpp.sh +++ b/ci/build_wheel_cpp.sh @@ -14,6 +14,10 @@ rapids-generate-version > ./VERSION cd "${package_dir}" python -m pip install wheel +# libkvikio is a header-only C++ library with no Python code, so +# it is entirely platform-agnostic. We cannot use auditwheel for +# retagging since it has no extension modules, so we use `wheel` +# directly instead. python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check python -m wheel tags --platform any dist/* --remove diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh index 1e6e4efac4..1fbe80800b 100755 --- a/ci/build_wheel_python.sh +++ b/ci/build_wheel_python.sh @@ -9,17 +9,14 @@ package_dir="python/kvikio" source rapids-configure-sccache source rapids-date-string - RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - rapids-generate-version > ./VERSION CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="libkvikio_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libkvikio_dist) cd "${package_dir}" -python -m pip install wheel python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check --find-links ${CPP_WHEELHOUSE} mkdir -p final_dist diff --git a/python/libkvikio/CMakeLists.txt b/python/libkvikio/CMakeLists.txt index 830cc5ea6f..278e09f462 100644 --- a/python/libkvikio/CMakeLists.txt +++ b/python/libkvikio/CMakeLists.txt @@ -16,9 +16,9 @@ cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) include(../../cpp/cmake/rapids_config.cmake) include(rapids-cpm) -rapids_cpm_init() include(rapids-cuda) rapids_cuda_init_architectures(libkvikio-python) +rapids_cpm_init() project( libkvikio-python