Add Python packages to the manifest

features/src/rapids-build-utils/.bashrc

@@ -1,8 +1,8 @@
 export CONDA_ALWAYS_YES="true";
 export CC="${CC:-"/usr/bin/gcc"}";
 export CXX="${CXX:-"/usr/bin/g++"}";
+export CUDAHOSTCXX="${CXX}";
 export CUDAARCHS="${CUDAARCHS:-all-major}";
-export CUDAHOSTCXX="${CUDAHOSTCXX:-"${CXX}"}";
 export CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}";
 export CMAKE_EXPORT_COMPILE_COMMANDS="${CMAKE_EXPORT_COMPILE_COMMANDS:-ON}";
 

features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh

@@ -91,11 +91,13 @@ generate_cpp_scripts() {
 
 generate_python_scripts() {
     local script_name;
-    for script_name in "build" "clean" "wheel"; do (
+    for script_name in "build" "clean"; do (
         cat ${TMPL}/python-${script_name}.tmpl.sh        \
       | generate_script "${script_name}-${PY_LIB}-python";
     ) || true;
     done
+    cat ${TMPL}/python-wheel.tmpl.sh       \
+  | generate_script "build-${PY_LIB}-wheel";
 }
 
 generate_scripts() {
@@ -128,6 +130,7 @@ generate_scripts() {
         local repo_name="${repo}_name";
         local repo_path="${repo}_path";
         local cpp_length="${repo}_cpp_length";
+        local py_length="${repo}_python_length";
         local git_repo="${repo}_git_repo";
         local git_host="${repo}_git_host";
         local git_tag="${repo}_git_tag";
@@ -209,8 +212,19 @@ generate_scripts() {
                 deps+=(-D$(tr "[:lower:]" "[:upper:]" <<< "${cpp_lib}")_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\");
             done
 
-            local py_libs=($(rapids-python-pkg-names "${!repo_path:-}" | tr "[:upper:]" "[:lower:]"));
-            local py_dirs=($(rapids-python-pkg-roots "${!repo_path:-}"));
+            local py_libs=()
+            local py_dirs=()
+
+            for ((j=0; j < ${!py_length:-0}; j+=1)); do
+                local py_name="${repo}_python_${j}_name";
+                local py_args="${repo}_python_${j}_args";
+                local py_sub_dir="${repo}_python_${j}_sub_dir";
+                local py_depends_length="${repo}_python_${j}_depends_length";
+                local py_path="${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}";
+
+                py_libs+=(${!py_name})
+                py_dirs+=($py_path)
+            done;
 
             for ((k=0; k < ${#py_libs[@]}; k+=1)); do
                 local py_dir="${py_dirs[$k]}";

features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh

@@ -32,7 +32,7 @@ list_repos() {
           }
         ]
         | map(select(.key | startswith("repos")))
-        | map(.key + "=" + (.val | @sh))[]
+        | map(.key + "=" + (.val | @sh | gsub("[\\n]"; " ")))[]
 ________EOF
 )";
 

features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh

@@ -1,21 +1,35 @@
 #! /usr/bin/env bash
 
-# Find the parent dir of the `setup.py` files in a RAPIDS library source tree.
+# list python package roots as determined by manifest.yaml with optional filter
 python_pkg_roots() {
     set -euo pipefail;
 
-    cd ~;
-    find "$@"                                          \
-        -type f                                        \
-     \( -name 'setup.py' -or -name 'pyproject.toml' \) \
-      ! -path '*conda*'                                \
-      ! -path '*build*'                                \
-        -exec sh -c "                                  \
-            grep -HP 'name(\s+)?=(\s+)?f?' {}          \
-          | head -n1" \;                               \
-      2>/dev/null                                      \
-      | sed -r 's@/(setup.py|pyproject.toml):.*$@@g'   \
-    ;
+    eval "$(                                  \
+        rapids-list-repos "$@"                \
+      | xargs -r -d'\n' -I% echo -n local %\; \
+    )";
+
+    for ((i=0; i < ${repos_length:-0}; i+=1)); do
+
+      local repo="repos_${i}";
+      local repo_path="${repo}_path";
+      local py_length="${repo}_python_length";
+
+      if [[ -d ~/"${!repo_path:-}/.git" ]]; then
+
+        for ((j=0; j < ${!py_length:-0}; j+=1)); do
+            local py_sub_dir="${repo}_python_${j}_sub_dir";
+            local py_path="${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}";
+
+            if [[ "$py_path" == ${1:-}* ]]; then
+              echo $py_path
+            fi
+
+        done;
+
+      fi;
+
+    done;
 }
 
 (python_pkg_roots "$@");

features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh

@@ -6,27 +6,34 @@ clone_${NAME}() {
     if [[ ! -d ~/"${SRC_PATH}"/.git ]]; then
 
         local branch=;
+        local upstream=;
+        local directory=;
 
         eval "$(                                  \
             devcontainer-utils-parse-args --names '
                 b|branch                          |
+                d|directory                       |
+                u|upstream                        |
             ' - <<< "$@"                          \
           | xargs -r -d'\n' -I% echo -n local %\; \
         )";
 
         branch="${b:-${branch:-"${GIT_TAG}"}}";
+        directory="${d:-${directory:-"${HOME}/${SRC_PATH}"}}";
+        upstream="${u:-${upstream:-"${GIT_UPSTREAM}/${GIT_REPO}"}}";
 
         echo 'Cloning ${NAME}' 1>&2;
 
-        __rest__+=(--tags);
-        __rest__+=(--recurse-submodules);
-        __rest__+=(-j $(nproc --ignore=2));
-        __rest__+=(-c checkout.defaultRemote=upstream);
-
         devcontainer-utils-clone-${GIT_HOST}-repo \
-            ${__rest__[@]} --branch "${branch}" \
-            "${GIT_UPSTREAM}/${GIT_REPO}" \
-            ~/"${SRC_PATH}";
+            --tags                                \
+            --branch "${branch}"                  \
+            --recurse-submodules                  \
+            -j $(nproc --ignore=2)                \
+            -c checkout.defaultRemote=upstream    \
+            ${__rest__[@]}                        \
+            "${upstream}"                         \
+            "${directory}"                        \
+        ;
 
         git -C ~/"${SRC_PATH}" config --add remote.upstream.fetch '^refs/heads/pull-request/*';
 

features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh

@@ -8,7 +8,7 @@ build_${PY_LIB}_python() {
         exit 1;
     fi
 
-    local verbose="1";
+    local verbose="";
     local parallel="";
 
     eval "$(                                  \

features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml

@@ -4,12 +4,17 @@ x-git-defaults: &git_defaults
     upstream: rapidsai
 
 repos:
+
 - name: rmm
   path: rmm
   cpp:
     - name: rmm
       sub_dir: ""
       depends: []
+  python:
+    - name: rmm
+      sub_dir: python
+      depends: []
   git:
     repo: rmm
     <<: *git_defaults
@@ -20,6 +25,10 @@ repos:
     - name: KvikIO
       sub_dir: cpp
       depends: []
+  python:
+    - name: kvikio
+      sub_dir: python
+      depends: []
   git:
     repo: kvikio
     <<: *git_defaults
@@ -32,7 +41,20 @@ repos:
       depends: [rmm, KvikIO]
     - name: cudf_kafka
       sub_dir: cpp/libcudf_kafka
-      depends: [rmm, KvikIO, cudf]
+      depends: [KvikIO, cudf]
+  python:
+    - name: cudf
+      sub_dir: python/cudf
+      depends: [rmm]
+    - name: dask_cudf
+      sub_dir: python/dask_cudf
+      depends: [cudf]
+    - name: cudf_kafka
+      sub_dir: python/cudf_kafka
+      depends: [cudf]
+    - name: custreamz
+      sub_dir: python/custreamz
+      depends: [cudf, cudf_kafka]
   git:
     repo: cudf
     <<: *git_defaults
@@ -43,6 +65,13 @@ repos:
     - name: raft
       sub_dir: cpp
       depends: [rmm]
+  python:
+    - name: pylibraft
+      sub_dir: python/pylibraft
+      depends: [rmm]
+    - name: raft-dask
+      sub_dir: python/raft-dask
+      depends: [pylibraft]
   git:
     repo: raft
     <<: *git_defaults
@@ -63,6 +92,10 @@ repos:
     - name: cuml
       sub_dir: cpp
       depends: [rmm, raft, cumlprims_mg]
+  python:
+    - name: cuml
+      sub_dir: python
+      depends: [rmm, cudf, dask-cudf, pylibraft, raft-dask]
   git:
     repo: cuml
     <<: *git_defaults
@@ -76,6 +109,13 @@ repos:
     - name: cugraph-ops-internal
       sub_dir: cpp_internal
       depends: [rmm, raft, cugraph-ops]
+  python:
+    - name: pylibcugraphops
+      sub_dir: pylibcugraphops
+      depends: []
+    - name: pylibcugraphops-internal
+      sub_dir: pylibcugraphops_internal
+      depends: [pylibcugraphops]
   git:
     repo: cugraph-ops
     <<: *git_defaults
@@ -86,6 +126,28 @@ repos:
     - name: cugraph
       sub_dir: cpp
       depends: [rmm, raft, cugraph-ops]
+  python:
+    - name: cugraph
+      sub_dir: python/cugraph
+      depends: [pylibraft, pylibcugraph]
+    - name: cugraph-dgl
+      sub_dir: python/cugraph-dgl
+      depends: []
+    - name: cugraph-nx
+      sub_dir: python/cugraph-nx
+      depends: []
+    - name: cugraph-pyg
+      sub_dir: python/cugraph-pyg
+      depends: []
+    - name: server
+      sub_dir: python/cugraph-service/client
+      depends: []
+    - name: client
+      sub_dir: python/cugraph-service/server
+      depends: []
+    - name: pylibcugraph
+      sub_dir: python/pylibcugraph
+      depends: [rmm, pylibraft]
   git:
     repo: cugraph
     <<: *git_defaults
@@ -96,6 +158,13 @@ repos:
     - name: cuspatial
       sub_dir: cpp
       depends: [rmm, cudf]
+  python:
+    - name: cuproj
+      sub_dir: python/cuproj
+      depends: [rmm]
+    - name: cuspatial
+      sub_dir: python/cuspatial
+      depends: [rmm, cudf]
   git:
     repo: cuspatial
     <<: *git_defaults

features/src/utils/opt/devcontainer/bin/git/repo/clone.sh

@@ -44,35 +44,28 @@ clone_git_repo() {
     fi
 
     if [ -n "${branch:-}" ]; then
-        has_branch() {
-            local b="${branch}";
-            for x in '\\' '\/' '.' '+' '^' '$' '*' '?' '!' '|' '=' '-' '[' ']' '(' ')' '{' '}'; do
-                b="${b//"$x"/\\"$x"}";
-            done
-            # git ls-remote exits with code 141, so compare the grep output and not just the exit code
-            if [ "$(git ls-remote "${1}" | grep -P "^[0-9a-z]{40}\s+refs\/heads\/${b}$" || echo "")" = "" ];
-            then echo "";
-            else echo "1";
-            fi
-        }
-
-        local origin_has_branch="$(has_branch "${origin}")";
-        local upstream_has_branch="$(has_branch "${upstream}")";
+
+        local origin_has_tag="$(git -C "${directory}" ls-remote -t origin "${branch}")";
+        local origin_has_branch="$(git -C "${directory}" ls-remote -h origin "${branch}")";
+        local upstream_has_tag="$(git -C "${directory}" ls-remote -t upstream "${branch}")";
+        local upstream_has_branch="$(git -C "${directory}" ls-remote -h upstream "${branch}")";
 
         if false; then exit 1;
-        elif [ -n "${upstream_has_branch}" ]; then
-            git -C "${directory}" checkout "${branch}" \
-         || git -C "${directory}" checkout "${branch}" -t "upstream/${branch}";
-        elif [ -n "${origin_has_branch}" ]; then
-            git -C "${directory}" checkout "${branch}" \
-         || git -C "${directory}" checkout "${branch}" -t "origin/${branch}";
+        elif test -n "${upstream_has_branch}"; then
+            git -C "${directory}" checkout -b "${branch}" -t "upstream/${branch}";
+        elif test -n "${upstream_has_tag}"; then
+            git -C "${directory}" checkout -m -b "upstream/${branch}" "${branch}";
+        elif test -n "${origin_has_branch}"; then
+            git -C "${directory}" checkout -b "${branch}" -t "origin/${branch}";
+        elif test -n "${origin_has_tag}"; then
+            git -C "${directory}" checkout -f -b "origin/${branch}" "${branch}";
         fi
 
-        if [ -n "${origin_has_branch}" ]; then
+        if test -n "${origin_has_branch}"; then
             git -C "${directory}" pull origin "${branch}";
         fi
 
-        if [ -n "${upstream_has_branch}" ]; then
+        if test -n "${upstream_has_branch}"; then
             git -C "${directory}" pull upstream "${branch}";
         fi
     fi

features/src/utils/opt/devcontainer/bin/github/repo/clone.sh

@@ -41,8 +41,9 @@ get_user_fork_name() {
             | map(.nameWithOwner)[]
 ________EOF
         )";
-        local nameWithOwner="$(gh repo list "${user}" --fork --json nameWithOwner --json parent --jq ". ${query}" 2>/dev/null || echo "")";
-        if test -z "${nameWithOwner}"; then
+        local nameWithOwner="$(gh repo list "${user}" --fork --json nameWithOwner --json parent --jq ". ${query}" 2>/dev/null || echo "err")";
+        if [ "${nameWithOwner}" = "err" ]; then
+            nameWithOwner="";
             for repo in $(gh repo list "${user}" --fork --json name --jq 'map(.name)[]'); do
                 nameWithOwner="$(gh repo view "${repo}" --json nameWithOwner --json parent --jq "[.] ${query}" 2>/dev/null || echo "")";
                 if test -n "${nameWithOwner}"; then