From b760453221c6eb5b6986ceb2153bb0a0284697e2 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Tue, 7 May 2024 22:08:55 -0700 Subject: [PATCH] Build C++ wheel (#2264) This PR changes wheel building in raft to create a separate C++ wheel that is then found from the Python wheel. The C++ wheel is now a hard dependency of the Python wheel. This allows Python packaging to more closely mirror the structure of our conda packaging, and the way we would normally wish to package these in any other manager. It also allows us to reduce package sizes by allowing better sharing of artifacts between different Python packages that rely on the same C++ components from other packages. Contributes to rapidsai/build-planning#33 Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ray Douglass (https://github.com/raydouglass) - Divye Gala (https://github.com/divyegala) URL: https://github.com/rapidsai/raft/pull/2264 --- .github/workflows/build.yaml | 30 ++++--- .github/workflows/pr.yaml | 27 +++--- build.sh | 12 +-- ci/build_wheel.sh | 67 --------------- ci/build_wheel_cpp.sh | 46 ++++++++++ ci/build_wheel_pylibraft.sh | 9 -- ci/build_wheel_python.sh | 84 +++++++++++++++++++ ci/build_wheel_raft_dask.sh | 9 -- ci/test_wheel_pylibraft.sh | 11 +-- ci/test_wheel_raft_dask.sh | 18 ++-- cpp/CMakeLists.txt | 69 +++++++++------ dependencies.yaml | 67 ++++++++++++++- python/libraft/CMakeLists.txt | 45 ++++++++++ python/libraft/LICENSE | 1 + python/libraft/README.md | 1 + python/libraft/libraft/VERSION | 1 + python/libraft/libraft/__init__.py | 17 ++++ python/libraft/libraft/_version.py | 25 ++++++ python/libraft/libraft/load.py | 48 +++++++++++ python/libraft/pyproject.toml | 65 ++++++++++++++ python/pylibraft/CMakeLists.txt | 39 +-------- python/pylibraft/pylibraft/__init__.py | 12 ++- .../pylibraft/cluster/CMakeLists.txt | 4 +- .../pylibraft/pylibraft/common/CMakeLists.txt | 4 +- .../pylibraft/distance/CMakeLists.txt | 2 +- .../pylibraft/pylibraft/matrix/CMakeLists.txt | 4 +- .../pylibraft/neighbors/CMakeLists.txt | 2 +- .../pylibraft/neighbors/cagra/CMakeLists.txt | 4 +- .../neighbors/ivf_flat/CMakeLists.txt | 4 +- .../pylibraft/neighbors/ivf_pq/CMakeLists.txt | 4 +- .../pylibraft/pylibraft/random/CMakeLists.txt | 4 +- python/pylibraft/pyproject.toml | 2 + python/raft-dask/CMakeLists.txt | 28 +------ python/raft-dask/pyproject.toml | 2 + python/raft-dask/raft_dask/__init__.py | 13 ++- .../raft-dask/raft_dask/common/CMakeLists.txt | 4 +- .../raft_dask/include_test/CMakeLists.txt | 4 +- 37 files changed, 537 insertions(+), 251 deletions(-) delete mode 100755 ci/build_wheel.sh create mode 100755 ci/build_wheel_cpp.sh delete mode 100755 ci/build_wheel_pylibraft.sh create mode 100755 ci/build_wheel_python.sh delete mode 100755 ci/build_wheel_raft_dask.sh create mode 100644 python/libraft/CMakeLists.txt create mode 120000 python/libraft/LICENSE create mode 120000 python/libraft/README.md create mode 120000 python/libraft/libraft/VERSION create mode 100644 python/libraft/libraft/__init__.py create mode 100644 python/libraft/libraft/_version.py create mode 100644 python/libraft/libraft/load.py create mode 100644 python/libraft/pyproject.toml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c8837afba7..d99d5d28e5 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,37 +67,40 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} - wheel-build-pylibraft: + wheel-build-cpp: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_pylibraft.sh - wheel-publish-pylibraft: - needs: wheel-build-pylibraft + script: ci/build_wheel_cpp.sh + wheel-build-python: + needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: pylibraft - wheel-build-raft-dask: - needs: wheel-publish-pylibraft + script: ci/build_wheel_python.sh + wheel-publish-cpp: + needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_raft_dask.sh - wheel-publish-raft-dask: - needs: wheel-build-raft-dask + package-name: raft + package-type: cpp + wheel-publish-python: + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: @@ -105,4 +108,5 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: raft_dask + package-name: raft + package-type: python diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c2d9556859..5d0368e3f7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,9 +19,9 @@ jobs: - conda-python-build - conda-python-tests - docs-build - - wheel-build-pylibraft + - wheel-build-cpp + - wheel-build-python - wheel-tests-pylibraft - - wheel-build-raft-dask - wheel-tests-raft-dask - devcontainer secrets: inherit @@ -74,29 +74,30 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-pylibraft: + wheel-build-cpp: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request - script: ci/build_wheel_pylibraft.sh - wheel-tests-pylibraft: - needs: wheel-build-pylibraft + script: ci/build_wheel_cpp.sh + wheel-build-python: + needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: pull-request - script: ci/test_wheel_pylibraft.sh - wheel-build-raft-dask: - needs: wheel-tests-pylibraft + script: ci/build_wheel_python.sh + wheel-tests-pylibraft: + needs: wheel-build-python secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request - script: "ci/build_wheel_raft_dask.sh" + script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: - needs: wheel-build-raft-dask + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: diff --git a/build.sh b/build.sh index da5efa5183..ee16bb4b1b 100755 --- a/build.sh +++ b/build.sh @@ -381,14 +381,6 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi -# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_RAFT_CPP=ON" -fi -# Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS -SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') - # If clean given, run it prior to any other steps if (( ${CLEAN} == 1 )); then # If the dirs to clean are mounted dirs in a container, the @@ -495,13 +487,13 @@ fi # Build and (optionally) install the pylibraft Python package if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then - SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ + CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/pylibraft fi # Build and (optionally) install the raft-dask Python package if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then - SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ + CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/raft-dask fi diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh deleted file mode 100755 index e3e7ce9c89..0000000000 --- a/ci/build_wheel.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -package_name=$1 -package_dir=$2 -underscore_package_name=$(echo "${package_name}" | tr "-" "_") - -# Clear out system ucx files to ensure that we're getting ucx from the wheel. -rm -rf /usr/lib64/ucx -rm -rf /usr/lib64/libuc* - -source rapids-configure-sccache -source rapids-date-string - -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${underscore_package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -if [[ ${package_name} == "raft-dask" ]]; then - sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/distributed-ucxx==(.*)\"/distributed-ucxx${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -else - sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -fi - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi - -cd "${package_dir}" - -# Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check - -mkdir -p final_dist -python -m auditwheel repair -w final_dist --exclude "libucp.so.0" dist/* - -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh new file mode 100755 index 0000000000..0e0417dc35 --- /dev/null +++ b/ci/build_wheel_cpp.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libraft" +package_dir="python/libraft" + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh deleted file mode 100755 index ec30a28b92..0000000000 --- a/ci/build_wheel_pylibraft.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -set -euo pipefail - -# Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" - -ci/build_wheel.sh pylibraft python/pylibraft diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh new file mode 100755 index 0000000000..ae15d3734b --- /dev/null +++ b/ci/build_wheel_python.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Clear out system ucx files to ensure that we're getting ucx from the wheel +# when building raft-dask. +rm -rf /usr/lib64/ucx +rm -rf /usr/lib64/libuc* + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +echo "${version}" > VERSION + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist) +PYTHON_WHEELHOUSE="${PWD}/dist/" +PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" +WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${CPP_WHEELHOUSE}") +mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" + +FIND_LINKS="" +# Iterate over the array +for wheelhouse in "${WHEELHOUSES[@]}"; do + FIND_LINKS+="--find-links ${wheelhouse} " +done + + +build_wheel () { + local package_name="${1}" + local underscore_package_name=$(echo "${package_name}" | tr "-" "_") + + local package_dir="python/${package_name}" + local pyproject_file="${package_dir}/pyproject.toml" + local version_file="${package_dir}/${underscore_package_name}/_version.py" + + sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + + sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + + for dep in rmm libraft pylibraft ucx-py distributed-ucxx; do + sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + done + + # dask-cuda & rapids-dask-dependency don't get a suffix, but they do get an alpha spec. + for dep in dask-cuda rapids-dask-dependency; do + sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} + done + + if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} + fi + + pushd "${package_dir}" + + python -m pip wheel . -w "${PYTHON_WHEELHOUSE}" -vvv --no-deps --disable-pip-version-check ${FIND_LINKS} + popd +} + +build_wheel pylibraft +build_wheel raft-dask + +python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libraft.so --exclude "libucp.so.0" "${PYTHON_WHEELHOUSE}"/* +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${PYTHON_AUDITED_WHEELHOUSE}" diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh deleted file mode 100755 index 5ae12303d0..0000000000 --- a/ci/build_wheel_raft_dask.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023, NVIDIA CORPORATION. - -set -euo pipefail - -# Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" - -ci/build_wheel.sh raft-dask python/raft-dask diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index b38f5a690b..230889ae82 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -3,11 +3,12 @@ set -euo pipefail -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/pylibraft*.whl)[test] +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" -python -m pytest ./python/pylibraft/pylibraft/test +python -m pip install "pylibraft-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" + +python -m pytest python/pylibraft/pylibraft/test diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index fe2d44f2b3..b2ec9a0c8b 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -3,21 +3,21 @@ set -euo pipefail -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# Download the pylibraft built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep -python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" -python -m pip install "raft_dask-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links dist/ +python -m pip install "raft-dask-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" + +test_dir="python/raft-dask/raft_dask/test" # rapids-logger "pytest raft-dask" -# python -m pytest ./python/raft-dask/raft_dask/test +# python -m pytest ${test_dir} # rapids-logger "pytest raft-dask (ucx-py only)" -# python -m pytest ./python/raft-dask/raft_dask/test --run_ucx +# python -m pytest ${test_dir} --run_ucx rapids-logger "pytest raft-dask (ucxx only)" -python -m pytest ./python/raft-dask/raft_dask/test --run_ucxx +python -m pytest ${test_dir} --run_ucxx diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 259d9fe428..88ccc27be0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -75,9 +75,12 @@ if((BUILD_TESTS ) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() -option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" +option(RAFT_COMPILE_LIBRARY "Enable building raft library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) +option(RAFT_COMPILE_DYNAMIC_ONLY "Only build the static library and skip the +static library. Has no effect if RAFT_COMPILE_LIBRARY is OFF" OFF +) if(BUILD_CPU_ONLY) set(BUILD_SHARED_LIBS OFF) @@ -582,17 +585,23 @@ if(RAFT_COMPILE_LIBRARY) ) add_library(raft_lib SHARED $) - add_library(raft_lib_static STATIC $) + + set(raft_lib_targets raft_lib) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + add_library(raft_lib_static STATIC $) + list(APPEND raft_lib_targets raft_lib_static) + endif() set_target_properties( - raft_lib raft_lib_static + ${raft_lib_targets} PROPERTIES OUTPUT_NAME raft BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) - foreach(target raft_lib raft_lib_static raft_objs) + list(APPEND raft_lib_targets raft_objs) + foreach(target IN LISTS raft_lib_targets) target_link_libraries( ${target} PUBLIC raft::raft @@ -617,20 +626,22 @@ target_link_libraries(raft_compiled INTERFACE raft::raft $ -) + target_link_libraries( + raft_compiled_static INTERFACE raft::raft $ + ) +endif() # ################################################################################################## # * raft_distributed ------------------------------------------------------------------------------- @@ -679,8 +690,12 @@ install( EXPORT raft-exports ) +set(raft_compiled_install_targets raft_compiled) +if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND raft_compiled_install_targets raft_compiled_static) +endif() install( - TARGETS raft_compiled raft_compiled_static + TARGETS ${raft_compiled_install_targets} DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -693,12 +708,14 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - install( - TARGETS raft_lib_static - DESTINATION ${lib_dir} - COMPONENT compiled-static - EXPORT raft-compiled-static-lib-exports - ) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + install( + TARGETS raft_lib_static + DESTINATION ${lib_dir} + COMPONENT compiled-static + EXPORT raft-compiled-static-lib-exports + ) + endif() install( DIRECTORY include/raft_runtime DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -769,8 +786,12 @@ endif() set(raft_components compiled distributed) set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) - list(APPEND raft_components compiled compiled-static) - list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) + list(APPEND raft_components compiled) + list(APPEND raft_export_sets raft-compiled-lib-exports) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND raft_components compiled-static) + list(APPEND raft_export_sets raft-compiled-static-lib-exports) + endif() endif() string( diff --git a/dependencies.yaml b/dependencies.yaml index a336aa1577..685d0a8fac 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -8,6 +8,7 @@ files: includes: - build - build_pylibraft + - cython_build - cuda - cuda_version - depends_on_cupy @@ -28,6 +29,7 @@ files: arch: [x86_64, aarch64] includes: - build + - cython_build - cuda - cuda_version - develop @@ -60,6 +62,25 @@ files: - docs - py_version - test_pylibraft + py_build_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: build-system + includes: + - build + - librmm + py_run_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: project + includes: + # This is really a build requirement for anything using libraft to build + # against, but is not required when _running_ with libraft. There isn't a + # great way to express that without separating libraft into libraft and + # libraft-dev packages, though. + - librmm py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -68,6 +89,7 @@ files: includes: - build - build_pylibraft + - cython_build py_run_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -92,6 +114,7 @@ files: table: build-system includes: - build + - cython_build - depends_on_ucx_build py_run_raft_dask: output: pyproject @@ -136,7 +159,6 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4 - - cython>=3.0.0 - ninja - output_types: [conda] packages: @@ -181,7 +203,33 @@ dependencies: packages: [nvcc_linux-64=11.2] - matrix: {cuda: "11.2", arch: aarch64} packages: [nvcc_linux-aarch64=11.2] - + cython_build: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cython>=3.0.0 + - output_types: [requirements, pyproject] + packages: + - libraft==24.6.* + librmm: + common: + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - librmm-cu12==24.6.* + - matrix: {cuda: "11.*"} + packages: + - librmm-cu11==24.6.* + - matrix: null + packages: + - librmm==24.6.* build_pylibraft: common: - output_types: [conda] @@ -431,10 +479,15 @@ dependencies: - matrix: {cuda: "12.*"} packages: - *rmm_cu12 + - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - *rmm_cu11 - - {matrix: null, packages: [*rmm_conda]} + - libraft-cu11==24.6.* + - matrix: null + packages: + - *rmm_conda + - libraft==24.6.* run_raft_dask: common: - output_types: [conda, pyproject] @@ -466,11 +519,17 @@ dependencies: packages: - &pylibraft_cu12 pylibraft-cu12==24.6.* - &ucx_py_cu12 ucx-py-cu12==0.38.* + - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - &pylibraft_cu11 pylibraft-cu11==24.6.* - &ucx_py_cu11 ucx-py-cu11==0.38.* - - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} + - libraft-cu11==24.6.* + - matrix: null + packages: + - *pylibraft_conda + - *ucx_py_conda + - libraft==24.6.* test_python_common: common: - output_types: [conda, requirements, pyproject] diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt new file mode 100644 index 0000000000..cc3b48d033 --- /dev/null +++ b/python/libraft/CMakeLists.txt @@ -0,0 +1,45 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +include(rapids-cuda) +rapids_cuda_init_architectures(libraft-python) + +project( + libraft-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +# Check if raft is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python raft package. +find_package(raft "${RAPIDS_VERSION}") + +if(raft_FOUND) + return() +endif() + +unset(raft_FOUND) + +set(BUILD_TESTS OFF) +set(BUILD_PRIMS_BENCH OFF) +set(BUILD_ANN_BENCH OFF) +set(RAFT_COMPILE_LIBRARY ON) +set(RAFT_COMPILE_DYNAMIC_ONLY ON) +set(CUDA_STATIC_RUNTIME ON) + +add_subdirectory(../../cpp raft-cpp) diff --git a/python/libraft/LICENSE b/python/libraft/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libraft/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libraft/README.md b/python/libraft/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libraft/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libraft/libraft/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py new file mode 100644 index 0000000000..2ba8e06d56 --- /dev/null +++ b/python/libraft/libraft/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from libraft._version import __git_commit__, __version__ +from libraft.load import load_library diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py new file mode 100644 index 0000000000..3e3792a85c --- /dev/null +++ b/python/libraft/libraft/_version.py @@ -0,0 +1,25 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import importlib.resources + +__version__ = ( + importlib.resources.files("libraft") + .joinpath("VERSION") + .read_text() + .strip() +) +__git_commit__ = "" diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py new file mode 100644 index 0000000000..fb7bf35274 --- /dev/null +++ b/python/libraft/libraft/load.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + + +def load_library(): + # Dynamically load libraft.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + libraft_lib = None + try: + libraft_lib = ctypes.CDLL("libraft.so", ctypes.RTLD_GLOBAL) + except OSError: + # If neither of these directories contain the library, we assume we are + # in an environment where the C++ library is already installed + # somewhere else and the CMake build of the libraft Python + # package was a no-op. Note that this approach won't work for + # real editable installs of the libraft package, but that's not a use + # case I think we need to support. scikit-build-core has limited + # support for importlib.resources so there isn't a clean way to support + # that case yet. + for lib_dir in ("lib", "lib64"): + if os.path.isfile( + lib := os.path.join( + os.path.dirname(__file__), lib_dir, "libraft.so" + ) + ): + libraft_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) + break + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libraft was loaded from. + return libraft_lib diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml new file mode 100644 index 0000000000..c5a76e363a --- /dev/null +++ b/python/libraft/pyproject.toml @@ -0,0 +1,65 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] + +requires = [ + "cmake>=3.26.4", + "librmm==24.6.*", + "ninja", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "scikit_build_core.build" + +[project] +name = "libraft" +dynamic = ["version"] +description = "RAFT: Reusable Algorithms Functions and other Tools" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.9" +classifiers = [ + "Intended Audience :: Developers", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] +dependencies = [ + "librmm==24.6.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/raft" +Documentation = "https://docs.rapids.ai/api/raft/stable/" + +[project.entry-points."cmake.prefix"] +libraft = "libraft" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.exclude = ["*tests*"] +sdist.reproducible = true +wheel.packages = ["libraft"] +wheel.install-dir = "libraft" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libraft/VERSION" +regex = "(?P.*)" diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 7a2d77041d..bba5549f62 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,42 +27,9 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - ON -) - -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) - if(NOT TARGET raft::raft_lib) - message( - FATAL_ERROR - "Building against a preexisting libraft library requires the compiled libraft to have been built!" - ) - - endif() -else() - set(raft_FOUND OFF) -endif() +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) include(rapids-cython-core) - -if(NOT raft_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_PRIMS_BENCH OFF) - set(BUILD_ANN_BENCH OFF) - set(RAFT_COMPILE_LIBRARY ON) - set(CUDA_STATIC_RUNTIME ON) - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - - # When building the C++ libraries from source we must copy libraft.so alongside the - # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft - # library, we shouldn't need this - set(cython_lib_dir pylibraft) - install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) -endif() - rapids_cython_init() add_subdirectory(pylibraft/common) @@ -71,7 +38,3 @@ add_subdirectory(pylibraft/matrix) add_subdirectory(pylibraft/neighbors) add_subdirectory(pylibraft/random) add_subdirectory(pylibraft/cluster) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") -endif() diff --git a/python/pylibraft/pylibraft/__init__.py b/python/pylibraft/pylibraft/__init__.py index 3b67a5f951..8aac8f93da 100644 --- a/python/pylibraft/pylibraft/__init__.py +++ b/python/pylibraft/pylibraft/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,3 +14,13 @@ # from pylibraft._version import __git_commit__, __version__ + +# If libraft was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index 7d6e05d918..06a639436a 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX cluster_ ) diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index 6ce1dfe347..d1c1acb3aa 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX common_ ) diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index 2530e07a98..ffcef45c32 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX distance_ ) diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt index ffba10dea9..07d35325a5 100644 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ b/python/pylibraft/pylibraft/matrix/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX matrix_ ) diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt index 069038a0e8..2a954183d3 100644 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt @@ -20,7 +20,7 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ ) add_subdirectory(cagra) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt index 441bb0b311..2df03c7b0b 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_cagra_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt index 8f395faec9..f50051ba23 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2023, NVIDIA CORPORATION. +# Copyright (c) 2023-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfflat_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt index e3d721a6ea..e57798fcc6 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfpq_ ) diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index fcc5ee6311..7d61855111 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX random_ ) diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index 3e8ca0b6d3..bf98d9b9c8 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -18,6 +18,7 @@ requires = [ "cmake>=3.26.4", "cuda-python>=11.7.1,<12.0a0", "cython>=3.0.0", + "libraft==24.6.*", "ninja", "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", @@ -36,6 +37,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", + "libraft==24.6.*", "numpy>=1.23,<2.0a0", "rmm==24.6.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 2c629f3b73..3218ea4d32 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -25,38 +25,12 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - OFF -) - rapids_cpm_init() # Once https://github.com/rapidsai/ucxx/issues/173 is resolved we can remove this. find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) -else() - set(raft_FOUND OFF) -endif() - -if(NOT raft_FOUND) - # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all - # library compilation and we don't need to install anything here. - set(BUILD_TESTS OFF) - set(BUILD_ANN_BENCH OFF) - set(BUILD_PRIMS_BENCH OFF) - set(RAFT_COMPILE_LIBRARIES OFF) - set(RAFT_COMPILE_DIST_LIBRARY OFF) - set(RAFT_COMPILE_NN_LIBRARY OFF) - set(CUDA_STATIC_RUNTIME ON) - set(RAFT_DASK_UCXX_STATIC ON) - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) - find_package(NCCL REQUIRED) -endif() +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) include(rapids-cython-core) rapids_cython_init() diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 0181bef4ce..53b1c51e83 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -18,6 +18,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.0", + "libraft==24.6.*", "libucx==1.15.0", "ninja", "scikit-build-core[pyproject]>=0.7.0", @@ -36,6 +37,7 @@ requires-python = ">=3.9" dependencies = [ "dask-cuda==24.6.*", "joblib>=0.11", + "libraft==24.6.*", "numba>=0.57", "numpy>=1.23,<2.0a0", "pylibraft==24.6.*", diff --git a/python/raft-dask/raft_dask/__init__.py b/python/raft-dask/raft_dask/__init__.py index 19a037ae75..13b9a96154 100644 --- a/python/raft-dask/raft_dask/__init__.py +++ b/python/raft-dask/raft_dask/__init__.py @@ -15,8 +15,17 @@ from raft_dask._version import __git_commit__, __version__ -# If libucx was installed as a wheel, we must request it to load the library symbols. -# Otherwise, we assume that the library was installed in a system path that ld can find. +# If libraft or libucx was installed as a wheel, we must request that those packages +# load the library symbols. Otherwise, we assume that the libraries were installed in a +# system path that ld can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft + try: import libucx except ModuleNotFoundError: diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 3798b5ac4b..49dee15d8f 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -15,6 +15,6 @@ set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft raft::distributed) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX ) diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index e588ce1d1e..8475bcaa93 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -1,5 +1,5 @@ # ============================================================================= -# Copyright (c) 2022, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except # in compliance with the License. You may obtain a copy of the License at @@ -15,6 +15,6 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX )