Adds updates to build wheel and conda packages for cugraph-nx (#3793)

closes #3771 
closes #3772 
closes #3770 

This PR adds updates to build new `cugraph-nx` wheels and conda packages, updates the CI scripts to test them, and also updates the auto-versioning script used during releases to also update `cugraph-nx`.

This PR also cleans up some tech debt in `dependencies.yaml` and various conda recipes (added missing meta-data).

Authors:
  - Rick Ratzel (https://github.com/rlratzel)

Approvers:
  - Ray Douglass (https://github.com/raydouglass)
  - Erik Welch (https://github.com/eriknw)
  - Alex Barghi (https://github.com/alexbarghi-nv)

URL: #3793

ci/build_python.sh

@@ -26,17 +26,29 @@ rapids-mamba-retry mambabuild \
   --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
   conda/recipes/cugraph
 
+# NOTE: nothing in cugraph-nx is CUDA-specific, but it is built on each CUDA
+# platform to ensure it is included in each set of artifacts, since test
+# scripts only install from one set of artifacts based on the CUDA version used
+# for the test run.
+rapids-mamba-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
+  conda/recipes/cugraph-nx
+
+# NOTE: nothing in the cugraph-service packages are CUDA-specific, but they are
+# built on each CUDA platform to ensure they are included in each set of
+# artifacts, since test scripts only install from one set of artifacts based on
+# the CUDA version used for the test run.
+rapids-mamba-retry mambabuild \
+  --no-test \
+  --channel "${CPP_CHANNEL}" \
+  --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
+  conda/recipes/cugraph-service
+
 RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}"
 
 if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then
-  # Only one CUDA configuration is needed, so we choose CUDA 11 arbitrarily.
-  # Nothing in the cugraph-service packages is CUDA-specific.
-  rapids-mamba-retry mambabuild \
-    --no-test \
-    --channel "${CPP_CHANNEL}" \
-    --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \
-    conda/recipes/cugraph-service
-
   # Only CUDA 11 is supported right now due to PyTorch requirement.
   rapids-mamba-retry mambabuild \
     --no-test \

ci/release/update-version.sh

@@ -61,6 +61,7 @@ sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugr
 sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/client/cugraph_service_client/__init__.py
 sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/server/cugraph_service_server/__init__.py
 sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/pylibcugraph/pylibcugraph/__init__.py
+sed_runner "s/__version__ = .*/__version__ = \"${NEXT_FULL_TAG}\"/g" python/cugraph-nx/cugraph_nx/__init__.py
 
 # Python pyproject.toml updates
 sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph/pyproject.toml
@@ -69,6 +70,7 @@ sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-pyg
 sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/client/pyproject.toml
 sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/server/pyproject.toml
 sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/pylibcugraph/pyproject.toml
+sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-nx/pyproject.toml
 
 # Wheel testing script
 sed_runner "s/branch-.*/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_cugraph.sh

ci/test_python.sh

@@ -34,6 +34,7 @@ rapids-mamba-retry install \
   libcugraph \
   pylibcugraph \
   cugraph \
+  cugraph-nx \
   cugraph-service-server \
   cugraph-service-client
 
@@ -88,6 +89,31 @@ pytest \
   cugraph/pytest-based/bench_algos.py
 popd
 
+rapids-logger "pytest cugraph-nx"
+pushd python/cugraph-nx/cugraph_nx
+pytest \
+  --capture=no \
+  --verbose \
+  --cache-clear \
+  --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-nx.xml" \
+  --cov-config=../../.coveragerc \
+  --cov=cugraph_nx \
+  --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/cugraph-nx-coverage.xml" \
+  --cov-report=term \
+  --benchmark-disable \
+  tests
+popd
+
+rapids-logger "pytest networkx using cugraph-nx backend"
+pushd python/cugraph-nx
+./run_nx_tests.sh
+# run_nx_tests.sh outputs coverage data, so check that total coverage is >0.0%
+# in case cugraph-nx failed to load but fallback mode allowed the run to pass.
+_coverage=$(coverage report|grep "^TOTAL")
+echo "cugraph-nx coverage from networkx tests: $_coverage"
+echo $_coverage | awk '{ if ($NF == "0.0%") exit 1 }'
+popd
+
 rapids-logger "pytest cugraph-service (single GPU)"
 pushd python/cugraph-service
 pytest \

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -38,13 +38,15 @@ dependencies:
 - nbsphinx
 - nccl>=2.9.9
 - networkx>=2.5.1
+- networkx>=3.0
 - ninja
 - notebook>=0.5.0
 - numba>=0.57
 - numpy>=1.21
 - numpydoc
 - nvcc_linux-64=11.8
 - openmpi
+- packaging>=21
 - pandas
 - pre-commit
 - pydata-sphinx-theme
@@ -53,6 +55,7 @@ dependencies:
 - pytest
 - pytest-benchmark
 - pytest-cov
+- pytest-mpl
 - pytest-xdist
 - python-louvain
 - raft-dask==23.10.*

conda/environments/all_cuda-120_arch-x86_64.yaml

@@ -38,12 +38,14 @@ dependencies:
 - nbsphinx
 - nccl>=2.9.9
 - networkx>=2.5.1
+- networkx>=3.0
 - ninja
 - notebook>=0.5.0
 - numba>=0.57
 - numpy>=1.21
 - numpydoc
 - openmpi
+- packaging>=21
 - pandas
 - pre-commit
 - pydata-sphinx-theme
@@ -52,6 +54,7 @@ dependencies:
 - pytest
 - pytest-benchmark
 - pytest-cov
+- pytest-mpl
 - pytest-xdist
 - python-louvain
 - raft-dask==23.10.*

conda/recipes/cugraph-dgl/meta.yaml

@@ -35,6 +35,7 @@ tests:
 
 about:
   home: https://rapids.ai/
+  dev_url: https://github.com/rapidsai/cugraph
   license: Apache-2.0
   license_file: ../../../LICENSE
   summary: cuGraph library

conda/recipes/cugraph-nx/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+# This assumes the script is executed from the root of the repo directory
+
+./build.sh cugraph-nx

conda/recipes/cugraph-nx/meta.yaml

@@ -0,0 +1,43 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') %}
+{% set minor_version = version.split('.')[0] + '.' + version.split('.')[1] %}
+{% set py_version = environ['CONDA_PY'] %}
+{% set date_string = environ['RAPIDS_DATE_STRING'] %}
+
+package:
+  name: cugraph-nx
+  version: {{ version }}
+
+source:
+  git_url: ../../..
+
+build:
+  number: {{ GIT_DESCRIBE_NUMBER }}
+  build:
+      number: {{ GIT_DESCRIBE_NUMBER }}
+      string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
+
+requirements:
+  host:
+    - python
+  run:
+    - pylibcugraph ={{ version }}
+    - networkx >=3.0
+    - cupy >=12.0.0
+    - python
+
+tests:
+  imports:
+    - cugraph_nx
+  commands:
+    - pip check
+  requires:
+    - pip
+
+about:
+  home: https://rapids.ai/
+  dev_url: https://github.com/rapidsai/cugraph
+  license: Apache-2.0
+  license_file: ../../../LICENSE
+  summary: cuGraph backend for GPU-accelerated NetworkX

conda/recipes/cugraph-pyg/meta.yaml

@@ -41,6 +41,7 @@ tests:
 
 about:
   home: https://rapids.ai/
+  dev_url: https://github.com/rapidsai/cugraph
   license: Apache-2.0
   license_file: ../../../LICENSE
   summary: cuGraph-pyg library

conda/recipes/cugraph-service/meta.yaml

@@ -19,6 +19,7 @@ outputs:
       string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
     about:
       home: https://rapids.ai/
+      dev_url: https://github.com/rapidsai/cugraph
       license: Apache-2.0
       license_family: APACHE
       license_file: ../../../LICENSE
@@ -39,6 +40,7 @@ outputs:
       string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
     about:
       home: https://rapids.ai/
+      dev_url: https://github.com/rapidsai/cugraph
       license: Apache-2.0
       license_family: APACHE
       license_file: ../../../LICENSE

conda/recipes/cugraph/meta.yaml

@@ -97,6 +97,7 @@ tests:
 
 about:
   home: https://rapids.ai/
+  dev_url: https://github.com/rapidsai/cugraph
   license: Apache-2.0
   license_file: ../../../LICENSE
   summary: cuGraph library

conda/recipes/libcugraph/meta.yaml

@@ -118,6 +118,7 @@ outputs:
         - ucx-proc=*=gpu
     about:
       home: https://rapids.ai/
+      dev_url: https://github.com/rapidsai/cugraph
       license: Apache-2.0
       license_file: ../../../LICENSE
       summary: libcugraph library
@@ -146,6 +147,7 @@ outputs:
         - librmm ={{ minor_version }}
     about:
       home: https://rapids.ai/
+      dev_url: https://github.com/rapidsai/cugraph
       license: Apache-2.0
       license_file: ../../../LICENSE
       summary: libcugraph_etl library
@@ -175,6 +177,7 @@ outputs:
         - gtest {{ gtest_version }}
     about:
       home: https://rapids.ai/
+      dev_url: https://github.com/rapidsai/cugraph
       license: Apache-2.0
       license_file: ../../../LICENSE
       summary: libcugraph test & benchmark executables

conda/recipes/pylibcugraph/meta.yaml

@@ -76,6 +76,7 @@ tests:
 
 about:
   home: https://rapids.ai/
+  dev_url: https://github.com/rapidsai/cugraph
   license: Apache-2.0
   license_file: ../../../LICENSE
   summary: pylibcugraph library

dependencies.yaml

@@ -14,12 +14,14 @@ files:
       - python_build_cythonize
       - python_run_cugraph
       - python_run_pylibcugraph
+      - python_run_cugraph_nx
       - python_run_cugraph_dgl
       - python_run_cugraph_pyg
       - test_notebook
       - test_python_common
       - test_python_cugraph
       - test_python_pylibcugraph
+      - test_python_cugraph_nx
   checks:
     output: none
     includes:
@@ -103,6 +105,29 @@ files:
     includes:
       - test_python_common
       - test_python_pylibcugraph
+  py_build_cugraph_nx:
+    output: pyproject
+    pyproject_dir: python/cugraph-nx
+    extras:
+      table: build-system
+    includes:
+      - python_build_wheel
+  py_run_cugraph_nx:
+    output: pyproject
+    pyproject_dir: python/cugraph-nx
+    extras:
+      table: project
+    includes:
+      - python_run_cugraph_nx
+  py_test_cugraph_nx:
+    output: pyproject
+    pyproject_dir: python/cugraph-nx
+    extras:
+      table: project.optional-dependencies
+      key: test
+    includes:
+      - test_python_common
+      - test_python_cugraph_nx
   py_build_cugraph_dgl:
     output: pyproject
     pyproject_dir: python/cugraph-dgl
@@ -166,9 +191,6 @@ files:
       table: project.optional-dependencies
       key: test
     includes:
-      # TODO: I think that the contents of the server's pyproject.toml
-      # dependencies were just copied from cugraph, so I'm not sure if this
-      # list is really minimal or if it is a superset.
       - test_python_common
       - test_python_cugraph
   cugraph_dgl_dev:
@@ -316,7 +338,7 @@ dependencies:
     common:
       - output_types: [conda, pyproject]
         packages:
-          - pylibcugraph==23.10.*
+          - &pylibcugraph pylibcugraph==23.10.*
   python_run_cugraph:
     common:
       - output_types: [conda, pyproject]
@@ -343,16 +365,28 @@ dependencies:
       - output_types: pyproject
         packages:
           - &cupy_pip cupy-cuda11x>=12.0.0
-            # cudf uses fsspec but is protocol independent. cugraph 
+            # cudf uses fsspec but is protocol independent. cugraph
             # dataset APIs require [http] extras for use with cudf.
           - fsspec[http]>=0.6.0
-          - pylibcugraph==23.10.*
+          - *pylibcugraph
   python_run_pylibcugraph:
     common:
       - output_types: [conda, pyproject]
         packages:
           - *pylibraft
           - *rmm
+  python_run_cugraph_nx:
+    common:
+      - output_types: [conda, pyproject]
+        packages:
+          - networkx>=3.0
+      - output_types: conda
+        packages:
+          - *cupy
+      - output_types: pyproject
+        packages:
+          - *cupy_pip
+          - *pylibcugraph
   python_run_cugraph_dgl:
     common:
       - output_types: [conda, pyproject]
@@ -428,8 +462,6 @@ dependencies:
     common:
       - output_types: [conda, pyproject]
         packages:
-          - networkx>=2.5.1
-          - *numpy
           - pandas
           - pytest
           - pytest-benchmark
@@ -440,13 +472,23 @@ dependencies:
     common:
       - output_types: [conda, pyproject]
         packages:
+          - networkx>=2.5.1
+          - *numpy
           - python-louvain
           - scikit-learn>=0.23.1
   test_python_pylibcugraph:
     common:
       - output_types: [conda, pyproject]
         packages:
           - *cudf
+          - *numpy
+  test_python_cugraph_nx:
+    common:
+      - output_types: [conda, pyproject]
+        packages:
+          - packaging>=21
+            # not needed by cugraph-nx tests, but is required for running networkx tests
+          - pytest-mpl
   cugraph_dgl_dev:
     common:
       - output_types: [conda]