Merge branch 'branch-24.12' of https://github.com/rapidsai/cugraph in…

…to hetero-c-plc
rapidsai · Nov 14, 2024 · c2559ad · c2559ad
2 parents c2783aa + c712a98
commit c2559ad
Show file tree

Hide file tree

Showing 335 changed files with 683 additions and 9,063 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -31,7 +31,7 @@ ENV PYTHONDONTWRITEBYTECODE="1"
 
 ENV SCCACHE_REGION="us-east-2"
 ENV SCCACHE_BUCKET="rapids-sccache-devs"
-ENV VAULT_HOST="https://vault.ops.k8s.rapids.ai"
+ENV AWS_ROLE_ARN="arn:aws:iam::279114543810:role/nv-gha-token-sccache-devs"
 ENV HISTFILE="/home/coder/.cache/._bash_history"
 
 # cugraph_pyg's setup.py needs this defined when building in a conda env

diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -113,66 +113,6 @@ jobs:
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
       package-name: cugraph
-  wheel-build-nx-cugraph:
-    needs: wheel-publish-pylibcugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type || 'branch' }}
-      branch: ${{ inputs.branch }}
-      sha: ${{ inputs.sha }}
-      date: ${{ inputs.date }}
-      script: ci/build_wheel_nx-cugraph.sh
-  wheel-publish-nx-cugraph:
-    needs: wheel-build-nx-cugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type || 'branch' }}
-      branch: ${{ inputs.branch }}
-      sha: ${{ inputs.sha }}
-      date: ${{ inputs.date }}
-      package-name: nx-cugraph
-  wheel-build-cugraph-dgl:
-    needs: wheel-publish-cugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type || 'branch' }}
-      branch: ${{ inputs.branch }}
-      sha: ${{ inputs.sha }}
-      date: ${{ inputs.date }}
-      script: ci/build_wheel_cugraph-dgl.sh
-  wheel-publish-cugraph-dgl:
-    needs: wheel-build-cugraph-dgl
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type || 'branch' }}
-      branch: ${{ inputs.branch }}
-      sha: ${{ inputs.sha }}
-      date: ${{ inputs.date }}
-      package-name: cugraph-dgl
-  wheel-build-cugraph-pyg:
-    needs: wheel-publish-cugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type || 'branch' }}
-      branch: ${{ inputs.branch }}
-      sha: ${{ inputs.sha }}
-      date: ${{ inputs.date }}
-      script: ci/build_wheel_cugraph-pyg.sh
-  wheel-publish-cugraph-pyg:
-    needs: wheel-build-cugraph-pyg
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: ${{ inputs.build_type || 'branch' }}
-      branch: ${{ inputs.branch }}
-      sha: ${{ inputs.sha }}
-      date: ${{ inputs.date }}
-      package-name: cugraph-pyg
   wheel-build-cugraph-equivariant:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]

diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -25,12 +25,6 @@ jobs:
       - wheel-tests-pylibcugraph
       - wheel-build-cugraph
       - wheel-tests-cugraph
-      - wheel-build-nx-cugraph
-      - wheel-tests-nx-cugraph
-      - wheel-build-cugraph-dgl
-      - wheel-tests-cugraph-dgl
-      - wheel-build-cugraph-pyg
-      - wheel-tests-cugraph-pyg
       - wheel-build-cugraph-equivariant
       - wheel-tests-cugraph-equivariant
       - devcontainer
@@ -55,16 +49,12 @@ jobs:
           - '!notebooks/**'
           - '!python/**'
           - '!readme_pages/**'
-          # TODO: Remove this before merging
-          - '!.github/**'
         test_notebooks:
           - '**'
           - '!.devcontainers/**'
           - '!CONTRIBUTING.md'
           - '!README.md'
           - '!docs/**'
-          # TODO: Remove this before merging
-          - '!.github/**'
         test_python:
           - '**'
           - '!.devcontainers/**'
@@ -73,8 +63,6 @@ jobs:
           - '!docs/**'
           - '!img/**'
           - '!notebooks/**'
-          # TODO: Remove this before merging
-          - '!.github/**'
   checks:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -173,53 +161,6 @@ jobs:
     with:
       build_type: pull-request
       script: ci/test_wheel_cugraph.sh
-  wheel-build-nx-cugraph:
-    needs: wheel-tests-pylibcugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/build_wheel_nx-cugraph.sh
-  wheel-tests-nx-cugraph:
-    needs: [wheel-build-nx-cugraph, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_nx-cugraph.sh
-  wheel-build-cugraph-dgl:
-    needs: wheel-tests-cugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/build_wheel_cugraph-dgl.sh
-  wheel-tests-cugraph-dgl:
-    needs: [wheel-build-cugraph-dgl, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_cugraph-dgl.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
-  wheel-build-cugraph-pyg:
-    needs: wheel-tests-cugraph
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: pull-request
-      script: ci/build_wheel_cugraph-pyg.sh
-  wheel-tests-cugraph-pyg:
-    needs: [wheel-build-cugraph-pyg, changed-files]
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
-    with:
-      build_type: pull-request
-      script: ci/test_wheel_cugraph-pyg.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
   wheel-build-cugraph-equivariant:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]

diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -58,35 +58,6 @@ jobs:
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
       script: ci/test_wheel_cugraph.sh
-  wheel-tests-nx-cugraph:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: nightly
-      branch: ${{ inputs.branch }}
-      date: ${{ inputs.date }}
-      sha: ${{ inputs.sha }}
-      script: ci/test_wheel_nx-cugraph.sh
-  wheel-tests-cugraph-dgl:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: nightly
-      branch: ${{ inputs.branch }}
-      date: ${{ inputs.date }}
-      sha: ${{ inputs.sha }}
-      script: ci/test_wheel_cugraph-dgl.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
-  wheel-tests-cugraph-pyg:
-    secrets: inherit
-    uses: rapidsai/shared-workflows/.github/workflows/[email protected]
-    with:
-      build_type: nightly
-      branch: ${{ inputs.branch }}
-      date: ${{ inputs.date }}
-      sha: ${{ inputs.sha }}
-      script: ci/test_wheel_cugraph-pyg.sh
-      matrix_filter: map(select(.ARCH == "amd64"))
   wheel-tests-cugraph-equivariant:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -55,7 +55,7 @@ repos:
               setup[.]cfg$
       - id: verify-alpha-spec
   - repo: https://github.com/rapidsai/dependency-file-generator
-    rev: v1.13.11
+    rev: v1.16.0
     hooks:
         - id: rapids-dependency-file-generator
           args: ["--clean"]

diff --git a/benchmarks/nx-cugraph/pytest-based/README.md b/benchmarks/nx-cugraph/pytest-based/README.md
@@ -10,6 +10,7 @@ Our current benchmarks provide the following datasets:
 | --------    | ------- | ------- | ------- |
 | netscience  | 1,461    | 5,484 | Yes |
 | email-Eu-core  | 1,005    | 25,571 | Yes |
+| amazon0302  | 262,111  | 1,234,877 | Yes |
 | cit-Patents  | 3,774,768    | 16,518,948 | Yes |
 | hollywood  | 1,139,905    | 57,515,616 | No |
 | soc-LiveJournal1  | 4,847,571    | 68,993,773 | Yes |
@@ -39,16 +40,8 @@ NOTE:
   ./run-main-benchmarks.sh
   ```
 
-#### 2. `get_graph_bench_dataset.py`
-This script downloads the specified dataset using `cugraph.datasets`.
-
-**Usage:**
-  ```bash
-  python get_graph_bench_dataset.py [dataset]
-  ```
-
-#### 3. `create_results_summary_page.py`
-This script is designed to be run after `run-gap-benchmarks.sh` in order to generate an HTML page displaying a results table comparing default NetworkX to nx-cugraph. The script also provides information about the current system, so it should be run on the machine on which benchmarks were run.
+#### 2. `create_results_summary_page.py`
+This script is designed to be run after `run-main-benchmarks.sh` in order to generate an HTML page displaying a results table comparing default NetworkX to nx-cugraph. The script also provides information about the current system, so it should be run on the machine on which benchmarks were run.
 
 **Usage:**
   ```bash

diff --git a/benchmarks/nx-cugraph/pytest-based/create_results_summary_page.py b/benchmarks/nx-cugraph/pytest-based/create_results_summary_page.py
@@ -157,7 +157,7 @@ def get_system_info():
 
     dataset_patt = re.compile(".*ds=([\w-]+).*")
     backend_patt = re.compile(".*backend=(\w+).*")
-    k_patt = re.compile(".*k=(10*).*")
+    k_patt = re.compile(".*k=(\d+).*")
 
     # Organize all benchmark runs by the following hierarchy: algo -> backend -> dataset
     benchmarks = get_all_benchmark_info()

diff --git a/build.sh b/build.sh
@@ -43,7 +43,6 @@ VALIDARGS="
    --pydevelop
    --allgpuarch
    --skip_cpp_tests
-   --without_cugraphops
    --cmake_default_generator
    --clean
    -h
@@ -74,7 +73,6 @@ HELP="$0 [<target> ...] [<flag> ...]
    --pydevelop                - install the Python packages in editable mode
    --allgpuarch               - build for all supported GPU architectures
    --skip_cpp_tests           - do not build the SG test binaries as part of the libcugraph and libcugraph_etl targets
-   --without_cugraphops       - do not build algos that require cugraph-ops
    --cmake_default_generator  - use the default cmake generator instead of ninja
    --clean                    - clean an individual target (note: to do a complete rebuild, use the clean target described above)
    -h                         - print this text
@@ -107,7 +105,6 @@ BUILD_CPP_TESTS=ON
 BUILD_CPP_MG_TESTS=OFF
 BUILD_CPP_MTMG_TESTS=OFF
 BUILD_ALL_GPU_ARCH=0
-BUILD_WITH_CUGRAPHOPS=ON
 CMAKE_GENERATOR_OPTION="-G Ninja"
 PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true"
 
@@ -170,9 +167,6 @@ fi
 if hasArg --skip_cpp_tests; then
     BUILD_CPP_TESTS=OFF
 fi
-if hasArg --without_cugraphops; then
-    BUILD_WITH_CUGRAPHOPS=OFF
-fi
 if hasArg cpp-mtmgtests; then
     BUILD_CPP_MTMG_TESTS=ON
 fi
@@ -268,7 +262,6 @@ if buildDefault || hasArg libcugraph || hasArg all; then
               -DBUILD_TESTS=${BUILD_CPP_TESTS} \
               -DBUILD_CUGRAPH_MG_TESTS=${BUILD_CPP_MG_TESTS} \
 	      -DBUILD_CUGRAPH_MTMG_TESTS=${BUILD_CPP_MTMG_TESTS} \
-	      -DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS} \
               ${CMAKE_GENERATOR_OPTION} \
               ${CMAKE_VERBOSE_OPTION}
         cmake --build "${LIBCUGRAPH_BUILD_DIR}" -j${PARALLEL_LEVEL} ${INSTALL_TARGET} ${VERBOSE_FLAG}
@@ -312,7 +305,7 @@ if buildDefault || hasArg pylibcugraph || hasArg all; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/pylibcugraph
     else
-        SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS}" \
+        SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \
             python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/pylibcugraph
     fi
 fi
@@ -322,7 +315,7 @@ if buildDefault || hasArg cugraph || hasArg all; then
     if hasArg --clean; then
         cleanPythonDir ${REPODIR}/python/cugraph
     else
-        SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS};-DUSE_CUGRAPH_OPS=${BUILD_WITH_CUGRAPHOPS}" \
+        SKBUILD_CMAKE_ARGS="${SKBUILD_EXTRA_CMAKE_ARGS}" \
             python ${PYTHON_ARGS_FOR_INSTALL} ${REPODIR}/python/cugraph
     fi
 fi

diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -17,4 +17,6 @@ rapids-logger "Begin cpp build"
 
 RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcugraph
 
+sccache --show-adv-stats
+
 rapids-upload-conda-to-s3 cpp
diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -20,28 +20,25 @@ export RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION)
 
 rapids-logger "Begin py build"
 
+sccache --zero-stats
+
 # TODO: Remove `--no-test` flags once importing on a CPU
 # node works correctly
 rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/pylibcugraph
 
+sccache --show-adv-stats
+sccache --zero-stats
+
 rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/cugraph
 
-# NOTE: nothing in nx-cugraph is CUDA-specific, but it is built on each CUDA
-# platform to ensure it is included in each set of artifacts, since test
-# scripts only install from one set of artifacts based on the CUDA version used
-# for the test run.
-rapids-conda-retry mambabuild \
-  --no-test \
-  --channel "${CPP_CHANNEL}" \
-  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-  conda/recipes/nx-cugraph
+sccache --show-adv-stats
 
 # NOTE: nothing in the cugraph-service packages are CUDA-specific, but they are
 # built on each CUDA platform to ensure they are included in each set of
@@ -53,29 +50,6 @@ rapids-conda-retry mambabuild \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/cugraph-service
 
-RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
-
-if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
-  # Only CUDA 11 is supported right now due to PyTorch requirement.
-  rapids-conda-retry mambabuild \
-    --no-test \
-    --channel "${CPP_CHANNEL}" \
-    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-    --channel pytorch \
-    --channel pytorch-nightly \
-    conda/recipes/cugraph-pyg
-
-  # Only CUDA 11 is supported right now due to PyTorch requirement.
-  rapids-conda-retry mambabuild \
-    --no-test \
-    --channel "${CPP_CHANNEL}" \
-    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-    --channel dglteam/label/th23_cu118 \
-    --channel pytorch \
-    --channel pytorch-nightly \
-    conda/recipes/cugraph-dgl
-fi
-
 rapids-conda-retry mambabuild \
   --no-test \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \