diff --git a/features/src/rapids-build-utils/.bashrc b/features/src/rapids-build-utils/.bashrc index 47ab091b..7b1032bf 100644 --- a/features/src/rapids-build-utils/.bashrc +++ b/features/src/rapids-build-utils/.bashrc @@ -1,8 +1,8 @@ export CONDA_ALWAYS_YES="true"; export CC="${CC:-"/usr/bin/gcc"}"; export CXX="${CXX:-"/usr/bin/g++"}"; +export CUDAHOSTCXX="${CXX}"; export CUDAARCHS="${CUDAARCHS:-all-major}"; -export CUDAHOSTCXX="${CUDAHOSTCXX:-"${CXX}"}"; export CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-Release}"; export CMAKE_EXPORT_COMPILE_COMMANDS="${CMAKE_EXPORT_COMPILE_COMMANDS:-ON}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh index 7394de64..00b1603a 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh @@ -91,11 +91,13 @@ generate_cpp_scripts() { generate_python_scripts() { local script_name; - for script_name in "build" "clean" "wheel"; do ( + for script_name in "build" "clean"; do ( cat ${TMPL}/python-${script_name}.tmpl.sh \ | generate_script "${script_name}-${PY_LIB}-python"; ) || true; done + cat ${TMPL}/python-wheel.tmpl.sh \ + | generate_script "build-${PY_LIB}-wheel"; } generate_scripts() { @@ -128,6 +130,7 @@ generate_scripts() { local repo_name="${repo}_name"; local repo_path="${repo}_path"; local cpp_length="${repo}_cpp_length"; + local py_length="${repo}_python_length"; local git_repo="${repo}_git_repo"; local git_host="${repo}_git_host"; local git_tag="${repo}_git_tag"; @@ -209,8 +212,19 @@ generate_scripts() { deps+=(-D$(tr "[:lower:]" "[:upper:]" <<< "${cpp_lib}")_ROOT=\"$(realpath -m ~/${cpp_dir}/build/latest)\"); done - local py_libs=($(rapids-python-pkg-names "${!repo_path:-}" | tr "[:upper:]" "[:lower:]")); - local py_dirs=($(rapids-python-pkg-roots "${!repo_path:-}")); + local py_libs=() + local py_dirs=() + + for ((j=0; j < ${!py_length:-0}; j+=1)); do + local py_name="${repo}_python_${j}_name"; + local py_args="${repo}_python_${j}_args"; + local py_sub_dir="${repo}_python_${j}_sub_dir"; + local py_depends_length="${repo}_python_${j}_depends_length"; + local py_path="${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}"; + + py_libs+=(${!py_name}) + py_dirs+=($py_path) + done; for ((k=0; k < ${#py_libs[@]}; k+=1)); do local py_dir="${py_dirs[$k]}"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh index a20a7e00..946dbb50 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/list-repos.sh @@ -32,7 +32,7 @@ list_repos() { } ] | map(select(.key | startswith("repos"))) - | map(.key + "=" + (.val | @sh))[] + | map(.key + "=" + (.val | @sh | gsub("[\\n]"; " ")))[] ________EOF )"; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh index 6ff508bf..d2a350e1 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/python-pkg-roots.sh @@ -1,21 +1,35 @@ #! /usr/bin/env bash -# Find the parent dir of the `setup.py` files in a RAPIDS library source tree. +# list python package roots as determined by manifest.yaml with optional filter python_pkg_roots() { set -euo pipefail; - cd ~; - find "$@" \ - -type f \ - \( -name 'setup.py' -or -name 'pyproject.toml' \) \ - ! -path '*conda*' \ - ! -path '*build*' \ - -exec sh -c " \ - grep -HP 'name(\s+)?=(\s+)?f?' {} \ - | head -n1" \; \ - 2>/dev/null \ - | sed -r 's@/(setup.py|pyproject.toml):.*$@@g' \ - ; + eval "$( \ + rapids-list-repos "$@" \ + | xargs -r -d'\n' -I% echo -n local %\; \ + )"; + + for ((i=0; i < ${repos_length:-0}; i+=1)); do + + local repo="repos_${i}"; + local repo_path="${repo}_path"; + local py_length="${repo}_python_length"; + + if [[ -d ~/"${!repo_path:-}/.git" ]]; then + + for ((j=0; j < ${!py_length:-0}; j+=1)); do + local py_sub_dir="${repo}_python_${j}_sub_dir"; + local py_path="${!repo_path:-}${!py_sub_dir:+/${!py_sub_dir}}"; + + if [[ "$py_path" == ${1:-}* ]]; then + echo $py_path + fi + + done; + + fi; + + done; } (python_pkg_roots "$@"); diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh index a7be9b3b..6a271ee6 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clone.tmpl.sh @@ -6,27 +6,34 @@ clone_${NAME}() { if [[ ! -d ~/"${SRC_PATH}"/.git ]]; then local branch=; + local upstream=; + local directory=; eval "$( \ devcontainer-utils-parse-args --names ' b|branch | + d|directory | + u|upstream | ' - <<< "$@" \ | xargs -r -d'\n' -I% echo -n local %\; \ )"; branch="${b:-${branch:-"${GIT_TAG}"}}"; + directory="${d:-${directory:-"${HOME}/${SRC_PATH}"}}"; + upstream="${u:-${upstream:-"${GIT_UPSTREAM}/${GIT_REPO}"}}"; echo 'Cloning ${NAME}' 1>&2; - __rest__+=(--tags); - __rest__+=(--recurse-submodules); - __rest__+=(-j $(nproc --ignore=2)); - __rest__+=(-c checkout.defaultRemote=upstream); - devcontainer-utils-clone-${GIT_HOST}-repo \ - ${__rest__[@]} --branch "${branch}" \ - "${GIT_UPSTREAM}/${GIT_REPO}" \ - ~/"${SRC_PATH}"; + --tags \ + --branch "${branch}" \ + --recurse-submodules \ + -j $(nproc --ignore=2) \ + -c checkout.defaultRemote=upstream \ + ${__rest__[@]} \ + "${upstream}" \ + "${directory}" \ + ; git -C ~/"${SRC_PATH}" config --add remote.upstream.fetch '^refs/heads/pull-request/*'; diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh index 45bbd8fa..b0acbf98 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh @@ -8,7 +8,7 @@ build_${PY_LIB}_python() { exit 1; fi - local verbose="1"; + local verbose=""; local parallel=""; eval "$( \ diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml index 07e2831f..14719963 100755 --- a/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml +++ b/features/src/rapids-build-utils/opt/rapids-build-utils/manifest.yaml @@ -4,12 +4,17 @@ x-git-defaults: &git_defaults upstream: rapidsai repos: + - name: rmm path: rmm cpp: - name: rmm sub_dir: "" depends: [] + python: + - name: rmm + sub_dir: python + depends: [] git: repo: rmm <<: *git_defaults @@ -20,6 +25,10 @@ repos: - name: KvikIO sub_dir: cpp depends: [] + python: + - name: kvikio + sub_dir: python + depends: [] git: repo: kvikio <<: *git_defaults @@ -32,7 +41,20 @@ repos: depends: [rmm, KvikIO] - name: cudf_kafka sub_dir: cpp/libcudf_kafka - depends: [rmm, KvikIO, cudf] + depends: [KvikIO, cudf] + python: + - name: cudf + sub_dir: python/cudf + depends: [rmm] + - name: dask_cudf + sub_dir: python/dask_cudf + depends: [cudf] + - name: cudf_kafka + sub_dir: python/cudf_kafka + depends: [cudf] + - name: custreamz + sub_dir: python/custreamz + depends: [cudf, cudf_kafka] git: repo: cudf <<: *git_defaults @@ -43,6 +65,13 @@ repos: - name: raft sub_dir: cpp depends: [rmm] + python: + - name: pylibraft + sub_dir: python/pylibraft + depends: [rmm] + - name: raft-dask + sub_dir: python/raft-dask + depends: [pylibraft] git: repo: raft <<: *git_defaults @@ -63,6 +92,10 @@ repos: - name: cuml sub_dir: cpp depends: [rmm, raft, cumlprims_mg] + python: + - name: cuml + sub_dir: python + depends: [rmm, cudf, dask-cudf, pylibraft, raft-dask] git: repo: cuml <<: *git_defaults @@ -76,6 +109,13 @@ repos: - name: cugraph-ops-internal sub_dir: cpp_internal depends: [rmm, raft, cugraph-ops] + python: + - name: pylibcugraphops + sub_dir: pylibcugraphops + depends: [] + - name: pylibcugraphops-internal + sub_dir: pylibcugraphops_internal + depends: [pylibcugraphops] git: repo: cugraph-ops <<: *git_defaults @@ -86,6 +126,28 @@ repos: - name: cugraph sub_dir: cpp depends: [rmm, raft, cugraph-ops] + python: + - name: cugraph + sub_dir: python/cugraph + depends: [pylibraft, pylibcugraph] + - name: cugraph-dgl + sub_dir: python/cugraph-dgl + depends: [] + - name: cugraph-nx + sub_dir: python/cugraph-nx + depends: [] + - name: cugraph-pyg + sub_dir: python/cugraph-pyg + depends: [] + - name: server + sub_dir: python/cugraph-service/client + depends: [] + - name: client + sub_dir: python/cugraph-service/server + depends: [] + - name: pylibcugraph + sub_dir: python/pylibcugraph + depends: [rmm, pylibraft] git: repo: cugraph <<: *git_defaults @@ -96,6 +158,13 @@ repos: - name: cuspatial sub_dir: cpp depends: [rmm, cudf] + python: + - name: cuproj + sub_dir: python/cuproj + depends: [rmm] + - name: cuspatial + sub_dir: python/cuspatial + depends: [rmm, cudf] git: repo: cuspatial <<: *git_defaults diff --git a/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh index f5897771..b91b9f0a 100755 --- a/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/git/repo/clone.sh @@ -44,35 +44,28 @@ clone_git_repo() { fi if [ -n "${branch:-}" ]; then - has_branch() { - local b="${branch}"; - for x in '\\' '\/' '.' '+' '^' '$' '*' '?' '!' '|' '=' '-' '[' ']' '(' ')' '{' '}'; do - b="${b//"$x"/\\"$x"}"; - done - # git ls-remote exits with code 141, so compare the grep output and not just the exit code - if [ "$(git ls-remote "${1}" | grep -P "^[0-9a-z]{40}\s+refs\/heads\/${b}$" || echo "")" = "" ]; - then echo ""; - else echo "1"; - fi - } - - local origin_has_branch="$(has_branch "${origin}")"; - local upstream_has_branch="$(has_branch "${upstream}")"; + + local origin_has_tag="$(git -C "${directory}" ls-remote -t origin "${branch}")"; + local origin_has_branch="$(git -C "${directory}" ls-remote -h origin "${branch}")"; + local upstream_has_tag="$(git -C "${directory}" ls-remote -t upstream "${branch}")"; + local upstream_has_branch="$(git -C "${directory}" ls-remote -h upstream "${branch}")"; if false; then exit 1; - elif [ -n "${upstream_has_branch}" ]; then - git -C "${directory}" checkout "${branch}" \ - || git -C "${directory}" checkout "${branch}" -t "upstream/${branch}"; - elif [ -n "${origin_has_branch}" ]; then - git -C "${directory}" checkout "${branch}" \ - || git -C "${directory}" checkout "${branch}" -t "origin/${branch}"; + elif test -n "${upstream_has_branch}"; then + git -C "${directory}" checkout -b "${branch}" -t "upstream/${branch}"; + elif test -n "${upstream_has_tag}"; then + git -C "${directory}" checkout -m -b "upstream/${branch}" "${branch}"; + elif test -n "${origin_has_branch}"; then + git -C "${directory}" checkout -b "${branch}" -t "origin/${branch}"; + elif test -n "${origin_has_tag}"; then + git -C "${directory}" checkout -f -b "origin/${branch}" "${branch}"; fi - if [ -n "${origin_has_branch}" ]; then + if test -n "${origin_has_branch}"; then git -C "${directory}" pull origin "${branch}"; fi - if [ -n "${upstream_has_branch}" ]; then + if test -n "${upstream_has_branch}"; then git -C "${directory}" pull upstream "${branch}"; fi fi diff --git a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh index 092f7249..b7db3395 100755 --- a/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh +++ b/features/src/utils/opt/devcontainer/bin/github/repo/clone.sh @@ -41,8 +41,9 @@ get_user_fork_name() { | map(.nameWithOwner)[] ________EOF )"; - local nameWithOwner="$(gh repo list "${user}" --fork --json nameWithOwner --json parent --jq ". ${query}" 2>/dev/null || echo "")"; - if test -z "${nameWithOwner}"; then + local nameWithOwner="$(gh repo list "${user}" --fork --json nameWithOwner --json parent --jq ". ${query}" 2>/dev/null || echo "err")"; + if [ "${nameWithOwner}" = "err" ]; then + nameWithOwner=""; for repo in $(gh repo list "${user}" --fork --json name --jq 'map(.name)[]'); do nameWithOwner="$(gh repo view "${repo}" --json nameWithOwner --json parent --jq "[.] ${query}" 2>/dev/null || echo "")"; if test -n "${nameWithOwner}"; then