From c712a9857abac9d033fda41278cc07ce51059d34 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 14 Nov 2024 11:56:25 -0600 Subject: [PATCH] remove cugraph-dgl and cugraph-pyg building/testing/publishing (#4752) Similar to #4748, but for `cugraph-dgl` and `cugraph-pyg`. This PR removes all building, publishing, and testing code related to `cugraph-dgl` and `cugraph-pyg`. Those projects will very soon start publishing packages from https://github.com/rapidsai/cugraph-gnn instead. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Ralph Liu (https://github.com/nv-rliu) - Bradley Dice (https://github.com/bdice) URL: https://github.com/rapidsai/cugraph/pull/4752 --- .github/workflows/build.yaml | 40 -------------- .github/workflows/pr.yaml | 36 ------------ .github/workflows/test.yaml | 20 ------- ci/build_python.sh | 23 -------- ci/build_wheel.sh | 4 +- ci/build_wheel_cugraph-dgl.sh | 6 -- ci/build_wheel_cugraph-pyg.sh | 6 -- ci/test_python.sh | 101 +--------------------------------- ci/test_wheel_cugraph-dgl.sh | 38 ------------- ci/test_wheel_cugraph-pyg.sh | 57 ------------------- 10 files changed, 3 insertions(+), 328 deletions(-) delete mode 100755 ci/build_wheel_cugraph-dgl.sh delete mode 100755 ci/build_wheel_cugraph-pyg.sh delete mode 100755 ci/test_wheel_cugraph-dgl.sh delete mode 100755 ci/test_wheel_cugraph-pyg.sh diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 96387b13d11..9a055d1dc51 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -113,46 +113,6 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cugraph - wheel-build-cugraph-dgl: - needs: wheel-publish-cugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - script: ci/build_wheel_cugraph-dgl.sh - wheel-publish-cugraph-dgl: - needs: wheel-build-cugraph-dgl - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - package-name: cugraph-dgl - wheel-build-cugraph-pyg: - needs: wheel-publish-cugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - script: ci/build_wheel_cugraph-pyg.sh - wheel-publish-cugraph-pyg: - needs: wheel-build-cugraph-pyg - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.12 - with: - build_type: ${{ inputs.build_type || 'branch' }} - branch: ${{ inputs.branch }} - sha: ${{ inputs.sha }} - date: ${{ inputs.date }} - package-name: cugraph-pyg wheel-build-cugraph-equivariant: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3797086ad5d..bac56fec9f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -25,10 +25,6 @@ jobs: - wheel-tests-pylibcugraph - wheel-build-cugraph - wheel-tests-cugraph - - wheel-build-cugraph-dgl - - wheel-tests-cugraph-dgl - - wheel-build-cugraph-pyg - - wheel-tests-cugraph-pyg - wheel-build-cugraph-equivariant - wheel-tests-cugraph-equivariant - devcontainer @@ -165,38 +161,6 @@ jobs: with: build_type: pull-request script: ci/test_wheel_cugraph.sh - wheel-build-cugraph-dgl: - needs: wheel-tests-cugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 - with: - build_type: pull-request - script: ci/build_wheel_cugraph-dgl.sh - wheel-tests-cugraph-dgl: - needs: [wheel-build-cugraph-dgl, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph-dgl.sh - matrix_filter: map(select(.ARCH == "amd64")) - wheel-build-cugraph-pyg: - needs: wheel-tests-cugraph - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 - with: - build_type: pull-request - script: ci/build_wheel_cugraph-pyg.sh - wheel-tests-cugraph-pyg: - needs: [wheel-build-cugraph-pyg, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph-pyg.sh - matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-equivariant: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4e4caf11500..245ac018cec 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -58,26 +58,6 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/test_wheel_cugraph.sh - wheel-tests-cugraph-dgl: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: ci/test_wheel_cugraph-dgl.sh - matrix_filter: map(select(.ARCH == "amd64")) - wheel-tests-cugraph-pyg: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - with: - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: ci/test_wheel_cugraph-pyg.sh - matrix_filter: map(select(.ARCH == "amd64")) wheel-tests-cugraph-equivariant: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 diff --git a/ci/build_python.sh b/ci/build_python.sh index 5ddd764737a..9a8f1227488 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -50,29 +50,6 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cugraph-service -RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" - -if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then - # Only CUDA 11 is supported right now due to PyTorch requirement. - rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel pytorch \ - --channel pytorch-nightly \ - conda/recipes/cugraph-pyg - - # Only CUDA 11 is supported right now due to PyTorch requirement. - rapids-conda-retry mambabuild \ - --no-test \ - --channel "${CPP_CHANNEL}" \ - --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ - --channel dglteam/label/th23_cu118 \ - --channel pytorch \ - --channel pytorch-nightly \ - conda/recipes/cugraph-dgl -fi - rapids-conda-retry mambabuild \ --no-test \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 8c4eb94cf86..3c89d63538c 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -30,9 +30,7 @@ python -m pip wheel \ sccache --show-adv-stats # pure-python packages should be marked as pure, and not have auditwheel run on them. -if [[ ${package_name} == "cugraph-dgl" ]] || \ - [[ ${package_name} == "cugraph-pyg" ]] || \ - [[ ${package_name} == "cugraph-equivariant" ]]; then +if [[ ${package_name} == "cugraph-equivariant" ]]; then RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist else case "${RAPIDS_CUDA_VERSION}" in diff --git a/ci/build_wheel_cugraph-dgl.sh b/ci/build_wheel_cugraph-dgl.sh deleted file mode 100755 index d62f810cba4..00000000000 --- a/ci/build_wheel_cugraph-dgl.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -./ci/build_wheel.sh cugraph-dgl python/cugraph-dgl diff --git a/ci/build_wheel_cugraph-pyg.sh b/ci/build_wheel_cugraph-pyg.sh deleted file mode 100755 index 97baa243f73..00000000000 --- a/ci/build_wheel_cugraph-pyg.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -euo pipefail - -./ci/build_wheel.sh cugraph-pyg python/cugraph-pyg diff --git a/ci/test_python.sh b/ci/test_python.sh index 57d60bf42ed..a3a177dcfc6 100755 --- a/ci/test_python.sh +++ b/ci/test_python.sh @@ -100,109 +100,12 @@ rapids-logger "pytest cugraph-service (single GPU)" --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-service-coverage.xml" \ --cov-report=term -if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - # we are only testing in a single cuda version - # because of pytorch and rapids compatibilty problems - rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_dgl - - # activate test_cugraph_dgl environment for dgl - set +u - conda activate test_cugraph_dgl - set -u - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - --channel conda-forge \ - --channel dglteam/label/th23_cu118 \ - --channel nvidia \ - "libcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "pylibcugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "pylibcugraphops=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "cugraph=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "cugraph-dgl=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - 'pytorch>=2.3,<2.4' \ - 'cuda-version=11.8' - - rapids-print-env - - rapids-logger "pytest cugraph_dgl (single GPU)" - ./ci/run_cugraph_dgl_pytests.sh \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-dgl.xml" \ - --cov-config=../../.coveragerc \ - --cov=cugraph_dgl \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-dgl-coverage.xml" \ - --cov-report=term - - # Reactivate the test environment back - set +u - conda deactivate - conda activate test - set -u - else - rapids-logger "skipping cugraph_dgl pytest on ARM64" - fi -else - rapids-logger "skipping cugraph_dgl pytest on CUDA!=11.8" -fi - -if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_pyg - - # Temporarily allow unbound variables for conda activation. - set +u - conda activate test_cugraph_pyg - set -u - - rapids-print-env - - # TODO re-enable logic once CUDA 12 is testable - #if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then - CONDA_CUDA_VERSION="11.8" - PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html" - #else - # CONDA_CUDA_VERSION="12.1" - # PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html" - #fi - - # Will automatically install built dependencies of cuGraph-PyG - rapids-mamba-retry install \ - --channel "${CPP_CHANNEL}" \ - --channel "${PYTHON_CHANNEL}" \ - "cugraph-pyg=${RAPIDS_VERSION_MAJOR_MINOR}.*" \ - "pytorch>=2.3,<2.4" \ - "ogb" - - rapids-print-env - - rapids-logger "pytest cugraph_pyg (single GPU)" - # rmat is not tested because of multi-GPU testing - ./ci/run_cugraph_pyg_pytests.sh \ - --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-pyg.xml" \ - --cov-config=../../.coveragerc \ - --cov=cugraph_pyg \ - --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-pyg-coverage.xml" \ - --cov-report=term - - # Reactivate the test environment back - set +u - conda deactivate - conda activate test - set -u - else - rapids-logger "skipping cugraph_pyg pytest on ARM64" - fi -else - rapids-logger "skipping cugraph_pyg pytest on CUDA!=11.8" -fi - # test cugraph-equivariant if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then if [[ "${RUNNER_ARCH}" != "ARM64" ]]; then - # Reuse cugraph-dgl's test env for cugraph-equivariant + rapids-mamba-retry env create --yes -f env.yaml -n test_cugraph_equivariant set +u - conda activate test_cugraph_dgl + conda activate test_cugraph_equivariant set -u rapids-mamba-retry install \ --channel "${CPP_CHANNEL}" \ diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh deleted file mode 100755 index d7558d43b6d..00000000000 --- a/ci/test_wheel_cugraph-dgl.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -eoxu pipefail - -package_name="cugraph-dgl" - -mkdir -p ./dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# Download the pylibcugraph, cugraph, and cugraph-dgl built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist - -# determine pytorch and DGL sources -PKG_CUDA_VER="$(echo ${CUDA_VERSION} | cut -d '.' -f1,2 | tr -d '.')" -PKG_CUDA_VER_MAJOR=${PKG_CUDA_VER:0:2} -if [[ "${PKG_CUDA_VER_MAJOR}" == "12" ]]; then - PYTORCH_CUDA_VER="121" -else - PYTORCH_CUDA_VER=$PKG_CUDA_VER -fi -PYTORCH_URL="https://download.pytorch.org/whl/cu${PYTORCH_CUDA_VER}" -DGL_URL="https://data.dgl.ai/wheels/torch-2.3/cu${PYTORCH_CUDA_VER}/repo.html" - -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install \ - -v \ - --extra-index-url "${PYTORCH_URL}" \ - --find-links "${DGL_URL}" \ - "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ - 'dgl==2.4.0' \ - 'torch>=2.3.0,<2.4' - -python -m pytest python/cugraph-dgl/tests diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh deleted file mode 100755 index 2f508ee830b..00000000000 --- a/ci/test_wheel_cugraph-pyg.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024, NVIDIA CORPORATION. - -set -eoxu pipefail - -package_name="cugraph-pyg" - -mkdir -p ./dist -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" - -# Download the pylibcugraph, cugraph, and cugraph-pyg built in the previous step -RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="cugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-deps -RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 ./dist - -# determine pytorch and pyg sources -if [[ "${CUDA_VERSION}" == "11.8.0" ]]; then - PYTORCH_URL="https://download.pytorch.org/whl/cu118" - PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu118.html" -else - PYTORCH_URL="https://download.pytorch.org/whl/cu121" - PYG_URL="https://data.pyg.org/whl/torch-2.3.0+cu121.html" -fi - -# echo to expand wildcard before adding `[extra]` requires for pip -python -m pip install \ - -v \ - --extra-index-url "${PYTORCH_URL}" \ - --find-links "${PYG_URL}" \ - "$(echo ./local-deps/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./local-deps/cugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ - "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ - 'ogb' \ - 'pyg_lib' \ - 'torch>=2.3.0,<2.4' \ - 'torch-geometric>=2.5,<2.6' \ - 'torch_scatter' \ - 'torch_sparse' - -# RAPIDS_DATASET_ROOT_DIR is used by test scripts -export RAPIDS_DATASET_ROOT_DIR="$(realpath datasets)" - -# Used to skip certain examples in CI due to memory limitations -export CI_RUN=1 - -rapids-logger "pytest cugraph-pyg (single GPU)" -pushd python/cugraph-pyg/cugraph_pyg -python -m pytest \ - --cache-clear \ - --benchmark-disable \ - tests -# Test examples -for e in "$(pwd)"/examples/*.py; do - rapids-logger "running example $e" - (yes || true) | python $e -done -popd