From e19fd6eb130f8f49c711cd9e74bb6b5490ebabbf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 10:42:47 -0500 Subject: [PATCH 1/6] build wheels without build isolation --- .github/workflows/pr.yaml | 123 -------------------------------------- ci/build_wheel.sh | 33 +++++++++- dependencies.yaml | 10 ++-- 3 files changed, 36 insertions(+), 130 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e18e82df0..3ec774ce6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,138 +12,15 @@ concurrency: jobs: pr-builder: needs: - - changed-files - - checks - - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - - conda-python-build - - conda-python-tests - - docs-build - - rust-build - wheel-build-cuvs - - wheel-tests-cuvs - - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} - changed-files: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 - with: - files_yaml: | - test_cpp: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!python/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - test_notebooks: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!rust/**' - - '!thirdparty/LICENSES/**' - test_python: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 - with: - enable_check_generated_files: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 - with: - build_type: pull-request - node_type: cpu16 - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: (void (thrust::|cub::)|raft_cutlass) - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 - with: - build_type: pull-request - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" - rust-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_rust.sh" wheel-build-cuvs: - needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cuvs.sh - wheel-tests-cuvs: - needs: [wheel-build-cuvs, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cuvs.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all --verbose; - sccache -s; diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index d1030276f..39a28cce3 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -12,6 +12,27 @@ source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +rapids-logger "Generating build requirements" +matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${underscore_package_name}" \ + --matrix "${matrix_selectors}" \ +| tee /tmp/requirements-build.txt + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_rapids_build_${underscore_package_name}" \ + --matrix "${matrix_selectors}" \ +| tee -a /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + rapids-generate-version > ./VERSION cd "${package_dir}" @@ -32,8 +53,16 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac -# Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +rapids-logger "Building '${package_name}' wheel" +python -m pip wheel \ + -w dist \ + -v \ + --no-build-isolation \ + --no-deps \ + --disable-pip-version-check \ + . + +sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* diff --git a/dependencies.yaml b/dependencies.yaml index a68a550bb..cf9b68c8a 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -74,14 +74,14 @@ files: - rapids_build - cuda - rust - py_build_py_cuvs: + py_build_cuvs: output: pyproject pyproject_dir: python/cuvs extras: table: build-system includes: - build - py_rapids_build_py_cuvs: + py_rapids_build_cuvs: output: pyproject pyproject_dir: python/cuvs extras: @@ -90,7 +90,7 @@ files: includes: - rapids_build - build_py_cuvs - py_run_py_cuvs: + py_run_cuvs: output: pyproject pyproject_dir: python/cuvs extras: @@ -99,7 +99,7 @@ files: - cuda_wheels - run_py_cuvs - depends_on_pylibraft - py_test_py_cuvs: + py_test_cuvs: output: pyproject pyproject_dir: python/cuvs extras: @@ -116,7 +116,7 @@ files: table: build-system includes: - rapids_build_setuptools - py_rapids_build_py_cuvs_bench: + py_rapids_build_cuvs_bench: output: pyproject pyproject_dir: python/cuvs_bench extras: From 4d9fbd8adf7cfe951127e70a7607947010dd27c8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 15:28:24 -0500 Subject: [PATCH 2/6] enable all CI --- .github/workflows/pr.yaml | 123 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3ec774ce6..e18e82df0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,15 +12,138 @@ concurrency: jobs: pr-builder: needs: + - changed-files + - checks + - conda-cpp-build + - conda-cpp-tests + - conda-cpp-checks + - conda-python-build + - conda-python-tests + - docs-build + - rust-build - wheel-build-cuvs + - wheel-tests-cuvs + - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} + changed-files: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + with: + files_yaml: | + test_cpp: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!python/**' + - '!rust/**' + - '!thirdparty/LICENSES/**' + test_notebooks: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!README.md' + - '!rust/**' + - '!thirdparty/LICENSES/**' + test_python: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!rust/**' + - '!thirdparty/LICENSES/**' + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 + with: + enable_check_generated_files: false + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 + with: + build_type: pull-request + node_type: cpu16 + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + with: + build_type: pull-request + enable_check_symbols: true + symbol_exclusions: (void (thrust::|cub::)|raft_cutlass) + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 + with: + build_type: pull-request + conda-python-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" + rust-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_rust.sh" wheel-build-cuvs: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cuvs.sh + wheel-tests-cuvs: + needs: [wheel-build-cuvs, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cuvs.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + build_command: | + sccache -z; + build-all --verbose; + sccache -s; From 87daa62e66d7c7cca975d4e46fea378acb14f2a3 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 14:22:24 -0500 Subject: [PATCH 3/6] print stats, new patterns, just run builds in CI --- .github/workflows/pr.yaml | 109 -------------------------------------- .pre-commit-config.yaml | 2 +- ci/build_cpp.sh | 4 ++ ci/build_python.sh | 10 ++++ ci/build_wheel.sh | 28 ++-------- ci/build_wheel_cuvs.sh | 23 +++++++- 6 files changed, 41 insertions(+), 135 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e18e82df0..986e09314 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,138 +12,29 @@ concurrency: jobs: pr-builder: needs: - - changed-files - - checks - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - conda-python-build - - conda-python-tests - - docs-build - - rust-build - wheel-build-cuvs - - wheel-tests-cuvs - - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} - changed-files: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 - with: - files_yaml: | - test_cpp: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!python/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - test_notebooks: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!rust/**' - - '!thirdparty/LICENSES/**' - test_python: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!rust/**' - - '!thirdparty/LICENSES/**' - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 - with: - enable_check_generated_files: false conda-cpp-build: - needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: pull-request node_type: cpu16 - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: (void (thrust::|cub::)|raft_cutlass) conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: pull-request - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" - rust-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_rust.sh" wheel-build-cuvs: - needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cuvs.sh - wheel-tests-cuvs: - needs: [wheel-build-cuvs, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cuvs.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all --verbose; - sccache -s; diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 439b42959..f4fdf202e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -113,7 +113,7 @@ repos: cpp/cmake/modules/FindAVX\.cmake| - id: verify-alpha-spec - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.13.11 + rev: v1.16.0 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 7bc0be5a7..db4c496cc 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -15,6 +15,10 @@ rapids-print-env rapids-logger "Begin cpp build" +sccache --zero-stats + RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcuvs +sccache --show-adv-stats + rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index deb67e91c..3241a2c2b 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -24,6 +24,8 @@ version=$(rapids-generate-version) export RAPIDS_PACKAGE_VERSION=${version} echo "${version}" > VERSION +sccache --zero-stats + # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-conda-retry mambabuild \ @@ -31,6 +33,9 @@ rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ conda/recipes/cuvs +sccache --show-adv-stats +sccache --zero-stats + # Build cuvs-bench for each cuda and python version rapids-conda-retry mambabuild \ --no-test \ @@ -38,6 +43,9 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cuvs-bench +sccache --show-adv-stats +sccache --zero-stats + # Build cuvs-bench-cpu only in CUDA 12 jobs since it only depends on python # version RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" @@ -47,6 +55,8 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "12" ]]; then --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cuvs-bench-cpu + + sccache --show-adv-stats fi rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 39a28cce3..9766965a5 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,34 +5,12 @@ set -euo pipefail package_name=$1 package_dir=$2 -underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -rapids-logger "Generating build requirements" -matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_build_${underscore_package_name}" \ - --matrix "${matrix_selectors}" \ -| tee /tmp/requirements-build.txt - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_rapids_build_${underscore_package_name}" \ - --matrix "${matrix_selectors}" \ -| tee -a /tmp/requirements-build.txt - -rapids-logger "Installing build requirements" -python -m pip install \ - -v \ - --prefer-binary \ - -r /tmp/requirements-build.txt - rapids-generate-version > ./VERSION cd "${package_dir}" @@ -54,10 +32,12 @@ case "${RAPIDS_CUDA_VERSION}" in esac rapids-logger "Building '${package_name}' wheel" + +sccache --zero-stats + python -m pip wheel \ -w dist \ -v \ - --no-build-isolation \ --no-deps \ --disable-pip-version-check \ . @@ -67,4 +47,4 @@ sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist diff --git a/ci/build_wheel_cuvs.sh b/ci/build_wheel_cuvs.sh index e03da9f19..aeac3188a 100755 --- a/ci/build_wheel_cuvs.sh +++ b/ci/build_wheel_cuvs.sh @@ -3,6 +3,27 @@ set -euo pipefail +package_name="cuvs" + +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" @@ -15,4 +36,4 @@ esac # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUVS_CPP=OFF${EXTRA_CMAKE_ARGS}" -ci/build_wheel.sh cuvs python/cuvs +ci/build_wheel.sh "${package_name}" python/cuvs From cb9117d7014cbb401f94daeddb1afdefce943dc7 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 16:42:07 -0500 Subject: [PATCH 4/6] run all CI --- .github/workflows/pr.yaml | 109 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 986e09314..e18e82df0 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,29 +12,138 @@ concurrency: jobs: pr-builder: needs: + - changed-files + - checks - conda-cpp-build + - conda-cpp-tests + - conda-cpp-checks - conda-python-build + - conda-python-tests + - docs-build + - rust-build - wheel-build-cuvs + - wheel-tests-cuvs + - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} + changed-files: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + with: + files_yaml: | + test_cpp: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!python/**' + - '!rust/**' + - '!thirdparty/LICENSES/**' + test_notebooks: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!README.md' + - '!rust/**' + - '!thirdparty/LICENSES/**' + test_python: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!rust/**' + - '!thirdparty/LICENSES/**' + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 + with: + enable_check_generated_files: false conda-cpp-build: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: pull-request node_type: cpu16 + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + with: + build_type: pull-request + enable_check_symbols: true + symbol_exclusions: (void (thrust::|cub::)|raft_cutlass) conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: pull-request + conda-python-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" + rust-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_rust.sh" wheel-build-cuvs: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cuvs.sh + wheel-tests-cuvs: + needs: [wheel-build-cuvs, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cuvs.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + build_command: | + sccache -z; + build-all --verbose; + sccache -s; From 11c5340987be54ada6ccdd8ebf7e1aa22431d562 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 16:43:48 -0500 Subject: [PATCH 5/6] revert unnecessary changes --- ci/build_wheel.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9766965a5..4994374a8 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string @@ -47,4 +48,4 @@ sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist From 07c8445f4b422053e5822b89a013c49bccc9535a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 29 Oct 2024 09:36:15 -0500 Subject: [PATCH 6/6] revert no-build-isolation stuff --- ci/build_wheel_cuvs.sh | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/ci/build_wheel_cuvs.sh b/ci/build_wheel_cuvs.sh index aeac3188a..e03da9f19 100755 --- a/ci/build_wheel_cuvs.sh +++ b/ci/build_wheel_cuvs.sh @@ -3,27 +3,6 @@ set -euo pipefail -package_name="cuvs" - -rapids-logger "Generating build requirements" - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_build_${package_name}" \ - --file-key "py_rapids_build_${package_name}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ -| tee /tmp/requirements-build.txt - -rapids-logger "Installing build requirements" -python -m pip install \ - -v \ - --prefer-binary \ - -r /tmp/requirements-build.txt - -# build with '--no-build-isolation', for better sccache hit rate -# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) -export PIP_NO_BUILD_ISOLATION=0 - case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" @@ -36,4 +15,4 @@ esac # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUVS_CPP=OFF${EXTRA_CMAKE_ARGS}" -ci/build_wheel.sh "${package_name}" python/cuvs +ci/build_wheel.sh cuvs python/cuvs