add args to generated rapids build scripts (#116)

* use updated feature variable names * add debug postfix to generated scripts * fix removing setuptools build dirs * always run python_version through cut * parse -j|--parallel and -v|--verbose arguments in generated configure and build scripts * bump feature versions * add generated script to build a python wheel * revert typo * mount in feature scripts * fix stdin argument parsing with spaces in values * only allow digits to follow an argument w/o a separating space * fix newline handling
rapidsai · Aug 22, 2023 · 7b6a152 · 7b6a152
1 parent 7bc1571
commit 7b6a152
Show file tree

Hide file tree

Showing 20 changed files with 311 additions and 62 deletions.
diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json
@@ -49,7 +49,9 @@
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.conda/unified,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.conda/unified,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.devcontainer/cuda11.8-conda/features/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.devcontainer/cuda11.8-conda/features/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent"
   ],
   "customizations": {
     "vscode": {

diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json
@@ -48,7 +48,9 @@
     "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.local/unified,target=/home/coder/.local,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.local/unified,target=/home/coder/.local,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.devcontainer/cuda11.8-pip/features/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.devcontainer/cuda11.8-pip/features/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent"
   ],
   "customizations": {
     "vscode": {

diff --git a/.devcontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/cuda12.0-conda/devcontainer.json
@@ -49,7 +49,9 @@
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.conda/unified,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.conda/unified,target=/home/coder/.conda/envs,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.devcontainer/cuda12.0-conda/features/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.devcontainer/cuda12.0-conda/features/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent"
   ],
   "customizations": {
     "vscode": {

diff --git a/.devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/cuda12.0-pip/devcontainer.json
@@ -48,7 +48,9 @@
     "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
-    "source=${localWorkspaceFolder}/../.local/unified,target=/home/coder/.local,type=bind,consistency=consistent"
+    "source=${localWorkspaceFolder}/../.local/unified,target=/home/coder/.local,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.devcontainer/cuda12.0-pip/features/utils/opt/devcontainer/bin,target=/opt/devcontainer/bin,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/.devcontainer/cuda12.0-pip/features/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils,type=bind,consistency=consistent"
   ],
   "customizations": {
     "vscode": {

diff --git a/features/src/rapids-build-utils/devcontainer-feature.json b/features/src/rapids-build-utils/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA RAPIDS devcontainer build utilities",
   "id": "rapids-build-utils",
-  "version": "23.10.3",
+  "version": "23.10.4",
   "description": "A feature to install the RAPIDS devcontainer build utilities",
   "containerEnv": {
     "BASH_ENV": "/etc/bash.bash_env"

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/generate-scripts.sh
@@ -91,7 +91,7 @@ generate_cpp_scripts() {
 
 generate_python_scripts() {
     local script_name;
-    for script_name in "build" "clean"; do (
+    for script_name in "build" "clean" "wheel"; do (
         cat ${TMPL}/python-${script_name}.tmpl.sh        \
       | generate_script "${script_name}-${PY_LIB}-python";
     ) || true;

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-jobs-and-archs.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/get-jobs-and-archs.sh
@@ -3,13 +3,24 @@
 jobs_and_archs() {
     set -euo pipefail
 
+    local parallel="";
+
+    eval "$(                                  \
+        devcontainer-utils-parse-args --names '
+            j|parallel                        |
+        ' - <<< "$@"                          \
+      | xargs -r -d'\n' -I% echo -n local %\; \
+    )";
+
+    parallel="${j:-${parallel:-}}";
+    if [ "${parallel:-}" = "true" ]; then parallel=""; fi
+
     local free_mem=$(free -g | head -n2 | tail -n1 | cut -d ':' -f2 | tr -s '[:space:]' | cut -d' ' -f7);
-    local max_cpus=$((free_mem / 4));
-    local all_cpus=${JOBS:-${PARALLEL_LEVEL:-$(nproc --ignore=2)}};
-    local n_jobs=$(( all_cpus < max_cpus ? all_cpus : max_cpus ));
+    local all_cpus="${parallel:-${JOBS:-${PARALLEL_LEVEL:-$(nproc --ignore=2)}}}";
+    local n_jobs=$(( all_cpus < (free_mem / 4) ? all_cpus : (free_mem / 4) ));
 
     local archs;
-    archs=$(rapids-parse-cmake-var-from-args CMAKE_CUDA_ARCHITECTURES "$@");
+    archs=$(rapids-parse-cmake-var-from-args CMAKE_CUDA_ARCHITECTURES "${__rest__[@]}");
     archs="${archs:-${CMAKE_CUDA_ARCHITECTURES:-${CUDAARCHS:-}}}";
 
     local n_archs=1;
@@ -43,4 +54,8 @@ jobs_and_archs() {
     echo "n_arch=${n_archs}";
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (jobs_and_archs "$@");
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-conda-env.sh
@@ -14,13 +14,10 @@ make_conda_env() {
     fi
 
     local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}}";
-    cuda_version="$(echo "${cuda_version}" | cut -d'.' -f3 --complement)";
+    cuda_version="$(cut -d'.' -f3 --complement <<< "${cuda_version}")";
 
-    local python_version="${PYTHON_VERSION:-}";
-
-    if [ -z "${python_version}" ]; then
-        python_version="$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)";
-    fi
+    local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
+    python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
 
     local new_env_path="$(realpath -m /tmp/${env_file_name})";
     local old_env_path="$(realpath -m ~/.conda/envs/${env_file_name})";

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/make-pip-env.sh
@@ -14,13 +14,10 @@ make_pip_env() {
     fi
 
     local cuda_version="${CUDA_VERSION:-${CUDA_VERSION_MAJOR}.${CUDA_VERSION_MINOR}}";
-    cuda_version="$(echo "${cuda_version}" | cut -d'.' -f3 --complement)";
+    cuda_version="$(cut -d'.' -f3 --complement <<< "${cuda_version}")";
 
-    local python_version="${PYTHON_VERSION:-}";
-
-    if [ -z "${python_version}" ]; then
-        python_version="$(python3 --version 2>&1 | cut -d' ' -f2 | cut -d'.' -f3 --complement)";
-    fi
+    local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
+    python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
 
     local new_env_path="$(realpath -m /tmp/${env_file_name})";
     local old_env_path="$(realpath -m ~/.local/share/venvs/${env_file_name})";

diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/build.tmpl.sh
@@ -14,4 +14,8 @@ build_${NAME}() {
     done
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (build_${NAME} "$@");
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/clean.tmpl.sh
@@ -14,4 +14,8 @@ clean_${NAME}() {
     done
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (clean_${NAME} "$@");
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/configure.tmpl.sh
@@ -9,4 +9,8 @@ configure_${NAME}() {
     done
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (configure_${NAME} "$@");
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-build.tmpl.sh
@@ -8,19 +8,35 @@ build_${CPP_LIB}_cpp() {
         exit 1;
     fi
 
-    configure-${CPP_LIB}-cpp "$@";
+    local verbose="";
 
     eval "$(                                  \
-        rapids-get-jobs-and-archs "$@"        \
+        devcontainer-utils-parse-args --names '
+            v|verbose                         |
+        ' - <<< "$@"                          \
       | xargs -r -d'\n' -I% echo -n local %\; \
     )";
 
-    time                                      \
-    JOBS="${n_jobs}"                          \
-    PARALLEL_LEVEL="${n_jobs}"                \
-    cmake --build ~/${CPP_SRC}/build/latest   \
-          --parallel ${n_jobs} \
-          --verbose;
+    verbose="${v:-${verbose:-}}";
+
+    if test -n "${verbose}"; then verbose="--log-level=VERBOSE"; fi
+
+    configure-${CPP_LIB}-cpp ${verbose} ${__rest__[@]};
+
+    eval "$(                                     \
+        rapids-get-jobs-and-archs ${__rest__[@]} \
+      | xargs -r -d'\n' -I% echo -n local %\;    \
+    )";
+
+    if test -n "${verbose}"; then verbose="--verbose"; fi
+
+    time                                        \
+    JOBS="${n_jobs}" PARALLEL_LEVEL="${n_jobs}" \
+    cmake --build ~/${CPP_SRC}/build/latest ${verbose} --parallel ${n_jobs};
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (build_${CPP_LIB}_cpp "$@");
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-clean.tmpl.sh
@@ -7,4 +7,8 @@ clean_${CPP_LIB}_cpp() {
     rm -rf ~/${CPP_SRC}/build/latest/*;
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (clean_${CPP_LIB}_cpp "$@");
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/cpp-configure.tmpl.sh
@@ -8,7 +8,18 @@ configure_${CPP_LIB}_cpp() {
         exit 1;
     fi
 
-    local build_type="$(rapids-parse-cmake-build-type "$@" | tr '[:upper:]' '[:lower:]')";
+    local parallel="";
+
+    eval "$(                                  \
+        devcontainer-utils-parse-args --names '
+            j|parallel                        |
+        ' - <<< "$@"                          \
+      | xargs -r -d'\n' -I% echo -n local %\; \
+    )";
+
+    parallel="${j:-${parallel:-}}";
+
+    local build_type="$(rapids-parse-cmake-build-type ${__rest__[@]} | tr '[:upper:]' '[:lower:]')";
     local binary_dir=~/${CPP_SRC}/build/${build_type};
     local source_dir=~/${CPP_SRC};
 
@@ -25,11 +36,11 @@ configure_${CPP_LIB}_cpp() {
     cmake_args+=(-B ${binary_dir});
     cmake_args+=(${CPP_DEPS});
     cmake_args+=(${CPP_ARGS});
-    cmake_args+=(${@});
+    cmake_args+=(${__rest__[@]});
 
-    eval "$(                                  \
-        rapids-get-jobs-and-archs "$@"        \
-      | xargs -r -d'\n' -I% echo -n local %\; \
+    eval "$(                                    \
+        rapids-get-jobs-and-archs -j${parallel} \
+      | xargs -r -d'\n' -I% echo -n local %\;   \
     )";
 
     time                                                     \
@@ -46,4 +57,8 @@ configure_${CPP_LIB}_cpp() {
     fi
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (configure_${CPP_LIB}_cpp "$@");
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-build.tmpl.sh
@@ -8,28 +8,55 @@ build_${PY_LIB}_python() {
         exit 1;
     fi
 
+    local verbose="1";
+    local parallel="";
+
+    eval "$(                                  \
+        devcontainer-utils-parse-args --names '
+            j|parallel                        |
+            v|verbose                         |
+        ' - <<< "$@"                          \
+      | xargs -r -d'\n' -I% echo -n local %\; \
+    )";
+
+    verbose="${v:-${verbose:-}}";
+    parallel="${j:-${parallel:-${JOBS:-${PARALLEL_LEVEL:-$(nproc --ignore=2)}}}}";
+
     local cmake_args=();
 
+    if test -n "${verbose}"; then
+        cmake_args+=("--log-level=VERBOSE");
+    fi
+
     # Define both lowercase and uppercase
     # `-DFIND_<lib>_CPP=ON` and `-DFIND_<LIB>_CPP=ON` because the RAPIDS
     # scikit-build CMakeLists.txt's aren't 100% consistent in the casing
     cmake_args+=(${CPP_DEPS});
     cmake_args+=(${CPP_ARGS});
-    cmake_args+=(${@});
+    cmake_args+=(${__rest__[@]});
 
     local ninja_args=();
-    ninja_args+=("-v");
-    ninja_args+=("-j${JOBS:-${PARALLEL_LEVEL:-$(nproc --ignore=2)}}");
+    if test -n "${verbose}"; then
+        ninja_args+=("-v");
+    fi
+    if test -n "${parallel}"; then
+        if [ "${parallel:-}" = "true" ]; then
+            parallel="";
+        fi
+        ninja_args+=("-j${parallel}");
+    fi
 
     local pip_args=();
-    pip_args+=("-vv");
+    if test -n "${verbose}"; then
+        pip_args+=("-vv");
+    fi
     pip_args+=("--no-build-isolation");
     pip_args+=("--no-deps");
     pip_args+=("--editable");
-    pip_args+=(~/${PY_SRC});
+    pip_args+=(~/"${PY_SRC}");
     cmake_args+=("-DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON");
 
-    trap "rm -rf ~/${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info" EXIT;
+    trap "rm -rf ~/'${PY_SRC}/$(echo "${PY_LIB}" | tr '-' '_').egg-info'" EXIT;
 
     time                                              \
     CMAKE_GENERATOR="Ninja"                           \
@@ -40,4 +67,8 @@ build_${PY_LIB}_python() {
     ;
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (build_${PY_LIB}_python "$@");
diff --git a/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh b/features/src/rapids-build-utils/opt/rapids-build-utils/bin/tmpl/python-clean.tmpl.sh
@@ -6,8 +6,19 @@ clean_${PY_LIB}_cpp() {
 
     local py_lib="$(tr '-' '_' <<< "${PY_LIB}")";
 
-    rm -rf ~/"${PY_SRC}"/{build,_skbuild};
-    rm -rf ~/"${PY_SRC}/${py_lib}.egg-info";
+    rm -rf ~/"${PY_SRC}"/{_skbuild,${py_lib}.egg-info};
+
+    local python_version="${PYTHON_VERSION:-$(python3 --version 2>&1 | cut -d' ' -f2)}";
+    python_version="$(cut -d'.' -f3 --complement <<< "${python_version}")";
+    python_version="${python_version/./}";
+
+    local dir;
+    for dir in lib temp dist; do
+        local slug="${dir}.$(uname -s)-$(uname -m)-cpython-${python_version/./}";
+        if test -d ~/"${PY_SRC}"/build/${slug,,}; then
+            rm -rf ~/"${PY_SRC}"/build/${slug,,};
+        fi
+    done
 
     if test -d ~/"${PY_SRC}/${PY_LIB}"/; then
         find ~/"${PY_SRC}/${PY_LIB}"/ -type f \
@@ -22,4 +33,8 @@ clean_${PY_LIB}_cpp() {
     fi
 }
 
+if test -n "${rapids_build_utils_debug:-}"; then
+    PS4="+ ${BASH_SOURCE[0]}:\${LINENO} "; set -x;
+fi
+
 (clean_${PY_LIB}_cpp "$@");