From d1218164a50c3dd5aacd7166078ced65f2d70f62 Mon Sep 17 00:00:00 2001 From: Vyas Ramasubramani Date: Fri, 10 May 2024 10:38:54 -0700 Subject: [PATCH] Revert "Revert "Build C++ wheel (#2264)" (#2305)" This reverts commit ea54c792b2b415b238cecf238f418aabb37836a2. --- .github/workflows/build.yaml | 28 ++++--- .github/workflows/pr.yaml | 27 +++--- build.sh | 12 +-- ci/build_wheel.sh | 67 --------------- ci/build_wheel_cpp.sh | 46 ++++++++++ ci/build_wheel_pylibraft.sh | 9 -- ci/build_wheel_python.sh | 84 +++++++++++++++++++ ci/build_wheel_raft_dask.sh | 9 -- ci/test_wheel_pylibraft.sh | 11 +-- ci/test_wheel_raft_dask.sh | 10 +-- cpp/CMakeLists.txt | 69 +++++++++------ dependencies.yaml | 67 ++++++++++++++- python/libraft/CMakeLists.txt | 45 ++++++++++ python/libraft/LICENSE | 1 + python/libraft/README.md | 1 + python/libraft/libraft/VERSION | 1 + python/libraft/libraft/__init__.py | 17 ++++ python/libraft/libraft/_version.py | 25 ++++++ python/libraft/libraft/load.py | 48 +++++++++++ python/libraft/pyproject.toml | 65 ++++++++++++++ python/pylibraft/CMakeLists.txt | 39 +-------- python/pylibraft/pylibraft/__init__.py | 10 +++ .../pylibraft/cluster/CMakeLists.txt | 2 +- .../pylibraft/pylibraft/common/CMakeLists.txt | 2 +- .../pylibraft/distance/CMakeLists.txt | 2 +- .../pylibraft/pylibraft/matrix/CMakeLists.txt | 2 +- .../pylibraft/neighbors/CMakeLists.txt | 2 +- .../pylibraft/neighbors/cagra/CMakeLists.txt | 2 +- .../neighbors/ivf_flat/CMakeLists.txt | 2 +- .../pylibraft/neighbors/ivf_pq/CMakeLists.txt | 2 +- .../pylibraft/pylibraft/random/CMakeLists.txt | 2 +- python/pylibraft/pyproject.toml | 2 + python/raft-dask/CMakeLists.txt | 28 +------ python/raft-dask/pyproject.toml | 2 + python/raft-dask/raft_dask/__init__.py | 13 ++- .../raft-dask/raft_dask/common/CMakeLists.txt | 2 +- .../raft_dask/include_test/CMakeLists.txt | 2 +- 37 files changed, 521 insertions(+), 237 deletions(-) delete mode 100755 ci/build_wheel.sh create mode 100755 ci/build_wheel_cpp.sh delete mode 100755 ci/build_wheel_pylibraft.sh create mode 100755 ci/build_wheel_python.sh delete mode 100755 ci/build_wheel_raft_dask.sh create mode 100644 python/libraft/CMakeLists.txt create mode 120000 python/libraft/LICENSE create mode 120000 python/libraft/README.md create mode 120000 python/libraft/libraft/VERSION create mode 100644 python/libraft/libraft/__init__.py create mode 100644 python/libraft/libraft/_version.py create mode 100644 python/libraft/libraft/load.py create mode 100644 python/libraft/pyproject.toml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index e013d4f1c5..9944dd5198 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,36 +67,39 @@ jobs: node_type: "gpu-v100-latest-1" run_script: "ci/build_docs.sh" sha: ${{ inputs.sha }} - wheel-build-pylibraft: + wheel-build-cpp: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_pylibraft.sh - wheel-publish-pylibraft: - needs: wheel-build-pylibraft + script: ci/build_wheel_cpp.sh + wheel-build-python: + needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: pylibraft - wheel-build-raft-dask: + script: ci/build_wheel_python.sh + wheel-publish-cpp: + needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - script: ci/build_wheel_raft_dask.sh - wheel-publish-raft-dask: - needs: wheel-build-raft-dask + package-name: raft + package-type: cpp + wheel-publish-python: + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.06 with: @@ -104,4 +107,5 @@ jobs: branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} date: ${{ inputs.date }} - package-name: raft_dask + package-name: raft + package-type: python diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c2d9556859..5d0368e3f7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,9 +19,9 @@ jobs: - conda-python-build - conda-python-tests - docs-build - - wheel-build-pylibraft + - wheel-build-cpp + - wheel-build-python - wheel-tests-pylibraft - - wheel-build-raft-dask - wheel-tests-raft-dask - devcontainer secrets: inherit @@ -74,29 +74,30 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-pylibraft: + wheel-build-cpp: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request - script: ci/build_wheel_pylibraft.sh - wheel-tests-pylibraft: - needs: wheel-build-pylibraft + script: ci/build_wheel_cpp.sh + wheel-build-python: + needs: wheel-build-cpp secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 with: build_type: pull-request - script: ci/test_wheel_pylibraft.sh - wheel-build-raft-dask: - needs: wheel-tests-pylibraft + script: ci/build_wheel_python.sh + wheel-tests-pylibraft: + needs: wheel-build-python secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.06 + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: build_type: pull-request - script: "ci/build_wheel_raft_dask.sh" + script: ci/test_wheel_pylibraft.sh wheel-tests-raft-dask: - needs: wheel-build-raft-dask + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: diff --git a/build.sh b/build.sh index da5efa5183..ee16bb4b1b 100755 --- a/build.sh +++ b/build.sh @@ -381,14 +381,6 @@ if [[ ${CMAKE_TARGET} == "" ]]; then CMAKE_TARGET="all" fi -# Append `-DFIND_RAFT_CPP=ON` to EXTRA_CMAKE_ARGS unless a user specified the option. -SKBUILD_EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" -if [[ "${EXTRA_CMAKE_ARGS}" != *"DFIND_RAFT_CPP"* ]]; then - SKBUILD_EXTRA_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS} -DFIND_RAFT_CPP=ON" -fi -# Replace spaces with semicolons in SKBUILD_EXTRA_CMAKE_ARGS -SKBUILD_EXTRA_CMAKE_ARGS=$(echo ${SKBUILD_EXTRA_CMAKE_ARGS} | sed 's/ /;/g') - # If clean given, run it prior to any other steps if (( ${CLEAN} == 1 )); then # If the dirs to clean are mounted dirs in a container, the @@ -495,13 +487,13 @@ fi # Build and (optionally) install the pylibraft Python package if (( ${NUMARGS} == 0 )) || hasArg pylibraft; then - SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ + CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/pylibraft fi # Build and (optionally) install the raft-dask Python package if (( ${NUMARGS} == 0 )) || hasArg raft-dask; then - SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \ + CMAKE_ARGS="${EXTRA_CMAKE_ARGS}" \ python -m pip install --no-build-isolation --no-deps ${REPODIR}/python/raft-dask fi diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh deleted file mode 100755 index e3e7ce9c89..0000000000 --- a/ci/build_wheel.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -package_name=$1 -package_dir=$2 -underscore_package_name=$(echo "${package_name}" | tr "-" "_") - -# Clear out system ucx files to ensure that we're getting ucx from the wheel. -rm -rf /usr/lib64/ucx -rm -rf /usr/lib64/libuc* - -source rapids-configure-sccache -source rapids-date-string - -version=$(rapids-generate-version) -git_commit=$(git rev-parse HEAD) - -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# This is the version of the suffix with a preceding hyphen. It's used -# everywhere except in the final wheel name. -PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" - -# Patch project metadata files to include the CUDA version suffix and version override. -pyproject_file="${package_dir}/pyproject.toml" -version_file="${package_dir}/${underscore_package_name}/_version.py" - -sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} -echo "${version}" > VERSION -sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} - -# For nightlies we want to ensure that we're pulling in alphas as well. The -# easiest way to do so is to augment the spec with a constraint containing a -# min alpha version that doesn't affect the version bounds but does allow usage -# of alpha versions for that dependency without --pre -alpha_spec='' -if ! rapids-is-release-build; then - alpha_spec=',>=0.0.0a0' -fi - -if [[ ${package_name} == "raft-dask" ]]; then - sed -r -i "s/pylibraft==(.*)\"/pylibraft${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/ucx-py==(.*)\"/ucx-py${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/dask-cuda==(.*)\"/dask-cuda==\1${alpha_spec}\"/g" ${pyproject_file} - sed -r -i "s/distributed-ucxx==(.*)\"/distributed-ucxx${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} -else - sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} -fi - -if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then - sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} - sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} -fi - -cd "${package_dir}" - -# Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check - -mkdir -p final_dist -python -m auditwheel repair -w final_dist --exclude "libucp.so.0" dist/* - -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh new file mode 100755 index 0000000000..0e0417dc35 --- /dev/null +++ b/ci/build_wheel_cpp.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright (c) 2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libraft" +package_dir="python/libraft" + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +# Patch project metadata files to include the CUDA version suffix and version override. +pyproject_file="${package_dir}/pyproject.toml" +version_file="${package_dir}/${package_name}/_version.py" + +sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} +echo "${version}" > VERSION +sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +sed -r -i "s/librmm(.*)\"/librmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check + +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh deleted file mode 100755 index 895c311f46..0000000000 --- a/ci/build_wheel_pylibraft.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -# Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" - -ci/build_wheel.sh pylibraft python/pylibraft diff --git a/ci/build_wheel_python.sh b/ci/build_wheel_python.sh new file mode 100755 index 0000000000..ae15d3734b --- /dev/null +++ b/ci/build_wheel_python.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +# Clear out system ucx files to ensure that we're getting ucx from the wheel +# when building raft-dask. +rm -rf /usr/lib64/ucx +rm -rf /usr/lib64/libuc* + +source rapids-configure-sccache +source rapids-date-string + +version=$(rapids-generate-version) +git_commit=$(git rev-parse HEAD) + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" + +# This is the version of the suffix with a preceding hyphen. It's used +# everywhere except in the final wheel name. +PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" + +echo "${version}" > VERSION + +# For nightlies we want to ensure that we're pulling in alphas as well. The +# easiest way to do so is to augment the spec with a constraint containing a +# min alpha version that doesn't affect the version bounds but does allow usage +# of alpha versions for that dependency without --pre +alpha_spec='' +if ! rapids-is-release-build; then + alpha_spec=',>=0.0.0a0' +fi + +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libraft_dist) +PYTHON_WHEELHOUSE="${PWD}/dist/" +PYTHON_AUDITED_WHEELHOUSE="${PWD}/final_dist/" +WHEELHOUSES=("${PYTHON_WHEELHOUSE}" "${CPP_WHEELHOUSE}") +mkdir -p "${PYTHON_AUDITED_WHEELHOUSE}" + +FIND_LINKS="" +# Iterate over the array +for wheelhouse in "${WHEELHOUSES[@]}"; do + FIND_LINKS+="--find-links ${wheelhouse} " +done + + +build_wheel () { + local package_name="${1}" + local underscore_package_name=$(echo "${package_name}" | tr "-" "_") + + local package_dir="python/${package_name}" + local pyproject_file="${package_dir}/pyproject.toml" + local version_file="${package_dir}/${underscore_package_name}/_version.py" + + sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file} + sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file} + + sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file} + + for dep in rmm libraft pylibraft ucx-py distributed-ucxx; do + sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file} + done + + # dask-cuda & rapids-dask-dependency don't get a suffix, but they do get an alpha spec. + for dep in dask-cuda rapids-dask-dependency; do + sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file} + done + + if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then + sed -i "s/cuda-python[<=>\.,0-9a]*/cuda-python>=12.0,<13.0a0/g" ${pyproject_file} + sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file} + fi + + pushd "${package_dir}" + + python -m pip wheel . -w "${PYTHON_WHEELHOUSE}" -vvv --no-deps --disable-pip-version-check ${FIND_LINKS} + popd +} + +build_wheel pylibraft +build_wheel raft-dask + +python -m auditwheel repair -w "${PYTHON_AUDITED_WHEELHOUSE}" --exclude libraft.so --exclude "libucp.so.0" "${PYTHON_WHEELHOUSE}"/* +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${PYTHON_AUDITED_WHEELHOUSE}" diff --git a/ci/build_wheel_raft_dask.sh b/ci/build_wheel_raft_dask.sh deleted file mode 100755 index feba2d7a5b..0000000000 --- a/ci/build_wheel_raft_dask.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023-2024, NVIDIA CORPORATION. - -set -euo pipefail - -# Set up skbuild options. Enable sccache in skbuild config options -export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF" - -ci/build_wheel.sh raft-dask python/raft-dask diff --git a/ci/test_wheel_pylibraft.sh b/ci/test_wheel_pylibraft.sh index b38f5a690b..230889ae82 100755 --- a/ci/test_wheel_pylibraft.sh +++ b/ci/test_wheel_pylibraft.sh @@ -3,11 +3,12 @@ set -euo pipefail -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install $(echo ./dist/pylibraft*.whl)[test] +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" -python -m pytest ./python/pylibraft/pylibraft/test +python -m pip install "pylibraft-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" + +python -m pytest python/pylibraft/pylibraft/test diff --git a/ci/test_wheel_raft_dask.sh b/ci/test_wheel_raft_dask.sh index b8ca54f6e8..bc3af96688 100755 --- a/ci/test_wheel_raft_dask.sh +++ b/ci/test_wheel_raft_dask.sh @@ -3,15 +3,13 @@ set -euo pipefail -mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -RAPIDS_PY_WHEEL_NAME="raft_dask_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist -# Download the pylibraft built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibraft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibraft-dep -python -m pip install --no-deps ./local-pylibraft-dep/pylibraft*.whl +WHEELHOUSE="${PWD}/dist/" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp "${WHEELHOUSE}" +RAPIDS_PY_WHEEL_NAME="raft_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python "${WHEELHOUSE}" -python -m pip install "raft_dask-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links dist/ +python -m pip install "raft-dask-${RAPIDS_PY_CUDA_SUFFIX}[test]>=0.0.0a0" --find-links "${WHEELHOUSE}" test_dir="python/raft-dask/raft_dask/test" diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 7270c5a12b..f7d1c9b119 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -75,9 +75,12 @@ if((BUILD_TESTS ) set(RAFT_COMPILE_LIBRARY_DEFAULT ON) endif() -option(RAFT_COMPILE_LIBRARY "Enable building raft shared library instantiations" +option(RAFT_COMPILE_LIBRARY "Enable building raft library instantiations" ${RAFT_COMPILE_LIBRARY_DEFAULT} ) +option(RAFT_COMPILE_DYNAMIC_ONLY "Only build the static library and skip the +static library. Has no effect if RAFT_COMPILE_LIBRARY is OFF" OFF +) if(BUILD_CPU_ONLY) set(BUILD_SHARED_LIBS OFF) @@ -582,17 +585,23 @@ if(RAFT_COMPILE_LIBRARY) ) add_library(raft_lib SHARED $) - add_library(raft_lib_static STATIC $) + + set(raft_lib_targets raft_lib) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + add_library(raft_lib_static STATIC $) + list(APPEND raft_lib_targets raft_lib_static) + endif() set_target_properties( - raft_lib raft_lib_static + ${raft_lib_targets} PROPERTIES OUTPUT_NAME raft BUILD_RPATH "\$ORIGIN" INSTALL_RPATH "\$ORIGIN" INTERFACE_POSITION_INDEPENDENT_CODE ON ) - foreach(target raft_lib raft_lib_static raft_objs) + list(APPEND raft_lib_targets raft_objs) + foreach(target IN LISTS raft_lib_targets) target_link_libraries( ${target} PUBLIC raft::raft @@ -617,20 +626,22 @@ target_link_libraries(raft_compiled INTERFACE raft::raft $ -) + target_link_libraries( + raft_compiled_static INTERFACE raft::raft $ + ) +endif() # ################################################################################################## # * raft_distributed ------------------------------------------------------------------------------- @@ -679,8 +690,12 @@ install( EXPORT raft-exports ) +set(raft_compiled_install_targets raft_compiled) +if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND raft_compiled_install_targets raft_compiled_static) +endif() install( - TARGETS raft_compiled raft_compiled_static + TARGETS ${raft_compiled_install_targets} DESTINATION ${lib_dir} COMPONENT raft EXPORT raft-compiled-exports @@ -693,12 +708,14 @@ if(TARGET raft_lib) COMPONENT compiled EXPORT raft-compiled-lib-exports ) - install( - TARGETS raft_lib_static - DESTINATION ${lib_dir} - COMPONENT compiled-static - EXPORT raft-compiled-static-lib-exports - ) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + install( + TARGETS raft_lib_static + DESTINATION ${lib_dir} + COMPONENT compiled-static + EXPORT raft-compiled-static-lib-exports + ) + endif() install( DIRECTORY include/raft_runtime DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} @@ -769,8 +786,12 @@ endif() set(raft_components compiled distributed) set(raft_export_sets raft-compiled-exports raft-distributed-exports) if(TARGET raft_lib) - list(APPEND raft_components compiled compiled-static) - list(APPEND raft_export_sets raft-compiled-lib-exports raft-compiled-static-lib-exports) + list(APPEND raft_components compiled) + list(APPEND raft_export_sets raft-compiled-lib-exports) + if(NOT RAFT_COMPILE_DYNAMIC_ONLY) + list(APPEND raft_components compiled-static) + list(APPEND raft_export_sets raft-compiled-static-lib-exports) + endif() endif() string( diff --git a/dependencies.yaml b/dependencies.yaml index 6ed26a3b14..3daa400444 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -8,6 +8,7 @@ files: includes: - build - build_pylibraft + - cython_build - cuda - cuda_version - depends_on_cupy @@ -28,6 +29,7 @@ files: arch: [x86_64, aarch64] includes: - build + - cython_build - cuda - cuda_version - develop @@ -60,6 +62,25 @@ files: - docs - py_version - test_pylibraft + py_build_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: build-system + includes: + - build + - librmm + py_run_libraft: + output: pyproject + pyproject_dir: python/libraft + extras: + table: project + includes: + # This is really a build requirement for anything using libraft to build + # against, but is not required when _running_ with libraft. There isn't a + # great way to express that without separating libraft into libraft and + # libraft-dev packages, though. + - librmm py_build_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -68,6 +89,7 @@ files: includes: - build - build_pylibraft + - cython_build py_run_pylibraft: output: pyproject pyproject_dir: python/pylibraft @@ -92,6 +114,7 @@ files: table: build-system includes: - build + - cython_build - depends_on_ucx_build py_run_raft_dask: output: pyproject @@ -136,7 +159,6 @@ dependencies: - output_types: [conda, requirements, pyproject] packages: - &cmake_ver cmake>=3.26.4 - - cython>=3.0.0 - ninja - output_types: [conda] packages: @@ -181,7 +203,33 @@ dependencies: packages: [nvcc_linux-64=11.2] - matrix: {cuda: "11.2", arch: aarch64} packages: [nvcc_linux-aarch64=11.2] - + cython_build: + common: + - output_types: [conda, requirements, pyproject] + packages: + - cython>=3.0.0 + - output_types: [requirements, pyproject] + packages: + - libraft==24.6.* + librmm: + common: + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: {cuda: "12.*"} + packages: + - librmm-cu12==24.6.* + - matrix: {cuda: "11.*"} + packages: + - librmm-cu11==24.6.* + - matrix: null + packages: + - librmm==24.6.* build_pylibraft: common: - output_types: [conda] @@ -431,10 +479,15 @@ dependencies: - matrix: {cuda: "12.*"} packages: - *rmm_cu12 + - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - *rmm_cu11 - - {matrix: null, packages: [*rmm_conda]} + - libraft-cu11==24.6.* + - matrix: null + packages: + - *rmm_conda + - libraft==24.6.* run_raft_dask: common: - output_types: [conda, pyproject] @@ -466,11 +519,17 @@ dependencies: packages: - &pylibraft_cu12 pylibraft-cu12==24.6.* - &ucx_py_cu12 ucx-py-cu12==0.38.* + - libraft-cu12==24.6.* - matrix: {cuda: "11.*"} packages: - &pylibraft_cu11 pylibraft-cu11==24.6.* - &ucx_py_cu11 ucx-py-cu11==0.38.* - - {matrix: null, packages: [*pylibraft_conda, *ucx_py_conda]} + - libraft-cu11==24.6.* + - matrix: null + packages: + - *pylibraft_conda + - *ucx_py_conda + - libraft==24.6.* test_python_common: common: - output_types: [conda, requirements, pyproject] diff --git a/python/libraft/CMakeLists.txt b/python/libraft/CMakeLists.txt new file mode 100644 index 0000000000..cc3b48d033 --- /dev/null +++ b/python/libraft/CMakeLists.txt @@ -0,0 +1,45 @@ +# ============================================================================= +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +include(rapids-cuda) +rapids_cuda_init_architectures(libraft-python) + +project( + libraft-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +# Check if raft is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python raft package. +find_package(raft "${RAPIDS_VERSION}") + +if(raft_FOUND) + return() +endif() + +unset(raft_FOUND) + +set(BUILD_TESTS OFF) +set(BUILD_PRIMS_BENCH OFF) +set(BUILD_ANN_BENCH OFF) +set(RAFT_COMPILE_LIBRARY ON) +set(RAFT_COMPILE_DYNAMIC_ONLY ON) +set(CUDA_STATIC_RUNTIME ON) + +add_subdirectory(../../cpp raft-cpp) diff --git a/python/libraft/LICENSE b/python/libraft/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libraft/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libraft/README.md b/python/libraft/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libraft/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libraft/libraft/VERSION b/python/libraft/libraft/VERSION new file mode 120000 index 0000000000..d62dc733ef --- /dev/null +++ b/python/libraft/libraft/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libraft/libraft/__init__.py b/python/libraft/libraft/__init__.py new file mode 100644 index 0000000000..2ba8e06d56 --- /dev/null +++ b/python/libraft/libraft/__init__.py @@ -0,0 +1,17 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from libraft._version import __git_commit__, __version__ +from libraft.load import load_library diff --git a/python/libraft/libraft/_version.py b/python/libraft/libraft/_version.py new file mode 100644 index 0000000000..3e3792a85c --- /dev/null +++ b/python/libraft/libraft/_version.py @@ -0,0 +1,25 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +import importlib.resources + +__version__ = ( + importlib.resources.files("libraft") + .joinpath("VERSION") + .read_text() + .strip() +) +__git_commit__ = "" diff --git a/python/libraft/libraft/load.py b/python/libraft/libraft/load.py new file mode 100644 index 0000000000..fb7bf35274 --- /dev/null +++ b/python/libraft/libraft/load.py @@ -0,0 +1,48 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + + +def load_library(): + # Dynamically load libraft.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + libraft_lib = None + try: + libraft_lib = ctypes.CDLL("libraft.so", ctypes.RTLD_GLOBAL) + except OSError: + # If neither of these directories contain the library, we assume we are + # in an environment where the C++ library is already installed + # somewhere else and the CMake build of the libraft Python + # package was a no-op. Note that this approach won't work for + # real editable installs of the libraft package, but that's not a use + # case I think we need to support. scikit-build-core has limited + # support for importlib.resources so there isn't a clean way to support + # that case yet. + for lib_dir in ("lib", "lib64"): + if os.path.isfile( + lib := os.path.join( + os.path.dirname(__file__), lib_dir, "libraft.so" + ) + ): + libraft_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) + break + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libraft was loaded from. + return libraft_lib diff --git a/python/libraft/pyproject.toml b/python/libraft/pyproject.toml new file mode 100644 index 0000000000..c5a76e363a --- /dev/null +++ b/python/libraft/pyproject.toml @@ -0,0 +1,65 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] + +requires = [ + "cmake>=3.26.4", + "librmm==24.6.*", + "ninja", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +build-backend = "scikit_build_core.build" + +[project] +name = "libraft" +dynamic = ["version"] +description = "RAFT: Reusable Algorithms Functions and other Tools" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.9" +classifiers = [ + "Intended Audience :: Developers", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] +dependencies = [ + "librmm==24.6.*", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/raft" +Documentation = "https://docs.rapids.ai/api/raft/stable/" + +[project.entry-points."cmake.prefix"] +libraft = "libraft" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.exclude = ["*tests*"] +sdist.reproducible = true +wheel.packages = ["libraft"] +wheel.install-dir = "libraft" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libraft/VERSION" +regex = "(?P.*)" diff --git a/python/pylibraft/CMakeLists.txt b/python/pylibraft/CMakeLists.txt index 7a2d77041d..bba5549f62 100644 --- a/python/pylibraft/CMakeLists.txt +++ b/python/pylibraft/CMakeLists.txt @@ -27,42 +27,9 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - ON -) - -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) - if(NOT TARGET raft::raft_lib) - message( - FATAL_ERROR - "Building against a preexisting libraft library requires the compiled libraft to have been built!" - ) - - endif() -else() - set(raft_FOUND OFF) -endif() +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS compiled) include(rapids-cython-core) - -if(NOT raft_FOUND) - set(BUILD_TESTS OFF) - set(BUILD_PRIMS_BENCH OFF) - set(BUILD_ANN_BENCH OFF) - set(RAFT_COMPILE_LIBRARY ON) - set(CUDA_STATIC_RUNTIME ON) - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - - # When building the C++ libraries from source we must copy libraft.so alongside the - # pairwise_distance and random Cython libraries TODO: when we have a single 'compiled' raft - # library, we shouldn't need this - set(cython_lib_dir pylibraft) - install(TARGETS raft_lib DESTINATION ${cython_lib_dir}) -endif() - rapids_cython_init() add_subdirectory(pylibraft/common) @@ -71,7 +38,3 @@ add_subdirectory(pylibraft/matrix) add_subdirectory(pylibraft/neighbors) add_subdirectory(pylibraft/random) add_subdirectory(pylibraft/cluster) - -if(DEFINED cython_lib_dir) - rapids_cython_add_rpath_entries(TARGET raft PATHS "${cython_lib_dir}") -endif() diff --git a/python/pylibraft/pylibraft/__init__.py b/python/pylibraft/pylibraft/__init__.py index b0869501f3..8aac8f93da 100644 --- a/python/pylibraft/pylibraft/__init__.py +++ b/python/pylibraft/pylibraft/__init__.py @@ -14,3 +14,13 @@ # from pylibraft._version import __git_commit__, __version__ + +# If libraft was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft diff --git a/python/pylibraft/pylibraft/cluster/CMakeLists.txt b/python/pylibraft/pylibraft/cluster/CMakeLists.txt index 562cff5098..06a639436a 100644 --- a/python/pylibraft/pylibraft/cluster/CMakeLists.txt +++ b/python/pylibraft/pylibraft/cluster/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX cluster_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX cluster_ ) diff --git a/python/pylibraft/pylibraft/common/CMakeLists.txt b/python/pylibraft/pylibraft/common/CMakeLists.txt index 53279bfaf7..d1c1acb3aa 100644 --- a/python/pylibraft/pylibraft/common/CMakeLists.txt +++ b/python/pylibraft/pylibraft/common/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX common_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX common_ ) diff --git a/python/pylibraft/pylibraft/distance/CMakeLists.txt b/python/pylibraft/pylibraft/distance/CMakeLists.txt index 2530e07a98..ffcef45c32 100644 --- a/python/pylibraft/pylibraft/distance/CMakeLists.txt +++ b/python/pylibraft/pylibraft/distance/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX distance_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX distance_ ) diff --git a/python/pylibraft/pylibraft/matrix/CMakeLists.txt b/python/pylibraft/pylibraft/matrix/CMakeLists.txt index 5b7803db00..07d35325a5 100644 --- a/python/pylibraft/pylibraft/matrix/CMakeLists.txt +++ b/python/pylibraft/pylibraft/matrix/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX matrix_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX matrix_ ) diff --git a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt index 069038a0e8..2a954183d3 100644 --- a/python/pylibraft/pylibraft/neighbors/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/CMakeLists.txt @@ -20,7 +20,7 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ ) add_subdirectory(cagra) diff --git a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt index 0939d7c5b3..2df03c7b0b 100644 --- a/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/cagra/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_cagra_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_cagra_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt index 37c57c45db..f50051ba23 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_flat/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfflat_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfflat_ ) diff --git a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt index af431adb16..e57798fcc6 100644 --- a/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt +++ b/python/pylibraft/pylibraft/neighbors/ivf_pq/CMakeLists.txt @@ -20,5 +20,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX neighbors_ivfpq_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX neighbors_ivfpq_ ) diff --git a/python/pylibraft/pylibraft/random/CMakeLists.txt b/python/pylibraft/pylibraft/random/CMakeLists.txt index 10ff776471..7d61855111 100644 --- a/python/pylibraft/pylibraft/random/CMakeLists.txt +++ b/python/pylibraft/pylibraft/random/CMakeLists.txt @@ -23,5 +23,5 @@ set(linked_libraries raft::raft raft::compiled) rapids_cython_create_modules( CXX SOURCE_FILES "${cython_sources}" - LINKED_LIBRARIES "${linked_libraries}" ASSOCIATED_TARGETS raft MODULE_PREFIX random_ + LINKED_LIBRARIES "${linked_libraries}" MODULE_PREFIX random_ ) diff --git a/python/pylibraft/pyproject.toml b/python/pylibraft/pyproject.toml index df1001538e..8feb600f11 100644 --- a/python/pylibraft/pyproject.toml +++ b/python/pylibraft/pyproject.toml @@ -18,6 +18,7 @@ requires = [ "cmake>=3.26.4", "cuda-python>=11.7.1,<12.0a0", "cython>=3.0.0", + "libraft==24.6.*", "ninja", "rmm==24.6.*", "scikit-build-core[pyproject]>=0.7.0", @@ -36,6 +37,7 @@ license = { text = "Apache 2.0" } requires-python = ">=3.9" dependencies = [ "cuda-python>=11.7.1,<12.0a0", + "libraft==24.6.*", "numpy>=1.23,<2.0a0", "rmm==24.6.*", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/raft-dask/CMakeLists.txt b/python/raft-dask/CMakeLists.txt index 2c629f3b73..3218ea4d32 100644 --- a/python/raft-dask/CMakeLists.txt +++ b/python/raft-dask/CMakeLists.txt @@ -25,38 +25,12 @@ project( LANGUAGES CXX CUDA ) -option(FIND_RAFT_CPP "Search for existing RAFT C++ installations before defaulting to local files" - OFF -) - rapids_cpm_init() # Once https://github.com/rapidsai/ucxx/issues/173 is resolved we can remove this. find_package(ucx REQUIRED) include(cmake/thirdparty/get_ucxx.cmake) -# If the user requested it we attempt to find RAFT. -if(FIND_RAFT_CPP) - find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) -else() - set(raft_FOUND OFF) -endif() - -if(NOT raft_FOUND) - # raft-dask doesn't actually use raft libraries, it just needs the headers, so we can turn off all - # library compilation and we don't need to install anything here. - set(BUILD_TESTS OFF) - set(BUILD_ANN_BENCH OFF) - set(BUILD_PRIMS_BENCH OFF) - set(RAFT_COMPILE_LIBRARIES OFF) - set(RAFT_COMPILE_DIST_LIBRARY OFF) - set(RAFT_COMPILE_NN_LIBRARY OFF) - set(CUDA_STATIC_RUNTIME ON) - set(RAFT_DASK_UCXX_STATIC ON) - - add_subdirectory(../../cpp raft-cpp EXCLUDE_FROM_ALL) - list(APPEND CMAKE_MODULE_PATH ${CMAKE_BINARY_DIR}/cmake/find_modules) - find_package(NCCL REQUIRED) -endif() +find_package(raft "${RAPIDS_VERSION}" REQUIRED COMPONENTS distributed) include(rapids-cython-core) rapids_cython_init() diff --git a/python/raft-dask/pyproject.toml b/python/raft-dask/pyproject.toml index 4718d51a35..37691cc01b 100644 --- a/python/raft-dask/pyproject.toml +++ b/python/raft-dask/pyproject.toml @@ -18,6 +18,7 @@ build-backend = "scikit_build_core.build" requires = [ "cmake>=3.26.4", "cython>=3.0.0", + "libraft==24.6.*", "libucx==1.15.0", "ninja", "scikit-build-core[pyproject]>=0.7.0", @@ -36,6 +37,7 @@ requires-python = ">=3.9" dependencies = [ "dask-cuda==24.6.*", "joblib>=0.11", + "libraft==24.6.*", "libucx>=1.15.0", "numba>=0.57", "numpy>=1.23,<2.0a0", diff --git a/python/raft-dask/raft_dask/__init__.py b/python/raft-dask/raft_dask/__init__.py index 19a037ae75..13b9a96154 100644 --- a/python/raft-dask/raft_dask/__init__.py +++ b/python/raft-dask/raft_dask/__init__.py @@ -15,8 +15,17 @@ from raft_dask._version import __git_commit__, __version__ -# If libucx was installed as a wheel, we must request it to load the library symbols. -# Otherwise, we assume that the library was installed in a system path that ld can find. +# If libraft or libucx was installed as a wheel, we must request that those packages +# load the library symbols. Otherwise, we assume that the libraries were installed in a +# system path that ld can find. +try: + import libraft +except ModuleNotFoundError: + pass +else: + libraft.load_library() + del libraft + try: import libucx except ModuleNotFoundError: diff --git a/python/raft-dask/raft_dask/common/CMakeLists.txt b/python/raft-dask/raft_dask/common/CMakeLists.txt index 65d5f06577..49dee15d8f 100644 --- a/python/raft-dask/raft_dask/common/CMakeLists.txt +++ b/python/raft-dask/raft_dask/common/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources comms_utils.pyx nccl.pyx) set(linked_libraries raft::raft raft::distributed) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX ) diff --git a/python/raft-dask/raft_dask/include_test/CMakeLists.txt b/python/raft-dask/raft_dask/include_test/CMakeLists.txt index 2ff1cd9150..8475bcaa93 100644 --- a/python/raft-dask/raft_dask/include_test/CMakeLists.txt +++ b/python/raft-dask/raft_dask/include_test/CMakeLists.txt @@ -15,6 +15,6 @@ set(cython_sources raft_include_test.pyx) set(linked_libraries raft::raft) rapids_cython_create_modules( - SOURCE_FILES "${cython_sources}" ASSOCIATED_TARGETS raft LINKED_LIBRARIES "${linked_libraries}" + SOURCE_FILES "${cython_sources}" LINKED_LIBRARIES "${linked_libraries}" CXX )