Install clang-format-17 via llvm feature (#193)

* only update alternatives of packages installed by this feature

* hack around devcontainers inability to define the same feature multiple times

* install clang-format-17 via llvm feature

* cleanup

* bump feature version

* add clang-format-17 to gcc-13 ubuntu23.04 container

* include clangd-17 in the clang tools

.devcontainer/cuda11.8-conda/devcontainer.json

@@ -10,10 +10,10 @@
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
-    "./features/rapids-build-utils": {}
+    "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
-    "./features/rapids-build-utils"
+    "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda11.8-envs} ${localWorkspaceFolder}/../{rmm,kvikio,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
   "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"],

.devcontainer/cuda11.8-conda/features

@@ -1 +1 @@
-../../features/src
+../../features

.devcontainer/cuda11.8-pip/devcontainer.json

@@ -10,12 +10,12 @@
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
-    "./features/cuda": {"installcuDNN": true},
-    "./features/rapids-build-utils": {}
+    "./features/src/cuda": {"installcuDNN": true},
+    "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
-    "./features/cuda",
-    "./features/rapids-build-utils"
+    "./features/src/cuda",
+    "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda11.8-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
   "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"],

.devcontainer/cuda11.8-pip/features

@@ -1 +1 @@
-../../features/src
+../../features

.devcontainer/cuda12.0-conda/devcontainer.json

@@ -10,10 +10,10 @@
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
-    "./features/rapids-build-utils": {}
+    "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
-    "./features/rapids-build-utils"
+    "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/${localWorkspaceFolderBasename}-cuda12.0-envs} ${localWorkspaceFolder}/../{rmm,kvikio,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
   "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"],

.devcontainer/cuda12.0-conda/features

@@ -1 +1 @@
-../../features/src
+../../features

.devcontainer/cuda12.0-pip/devcontainer.json

@@ -10,12 +10,12 @@
   },
   "hostRequirements": {"gpu": "optional"},
   "features": {
-    "./features/cuda": {"installcuDNN": true},
-    "./features/rapids-build-utils": {}
+    "./features/src/cuda": {"installcuDNN": true},
+    "./features/src/rapids-build-utils": {}
   },
   "overrideFeatureInstallOrder": [
-    "./features/cuda",
-    "./features/rapids-build-utils"
+    "./features/src/cuda",
+    "./features/src/rapids-build-utils"
   ],
   "initializeCommand": ["/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/share/${localWorkspaceFolderBasename}-cuda12.0-venvs} ${localWorkspaceFolder}/../{rmm,kvikio,cudf,raft,cumlprims_mg,cuml,cugraph-ops,wholegraph,cugraph,cuspatial}"],
   "postAttachCommand": ["/bin/bash", "-c", "if [ ${CODESPACES:-false} = 'true' ]; then . devcontainer-utils-post-attach-command; . rapids-post-attach-command; fi"],

.devcontainer/cuda12.0-pip/features

@@ -1 +1 @@
-../../features/src
+../../features

.github/actions/devcontainer-json/action.sh

@@ -23,18 +23,30 @@ echo "tag=${tag}" >&3;
 
 node -e "$(cat <<EOF
 
-const json = JSON.parse(require('fs').readFileSync('image/.devcontainer/devcontainer.json'));
+const {cpSync, readFileSync} = require('fs');
+const json = JSON.parse(readFileSync('image/.devcontainer/devcontainer.json'));
 
 json.build.args.BASE = '${os}';
 json.containerEnv = ${container_env} || undefined;
 
+const dups = {};
+
 ${features}.forEach(({name, ...feature}) => {
   const i = json.overrideFeatureInstallOrder.length - 1;
   if (name.includes(':')) {
     json.features[name] = feature;
     json.overrideFeatureInstallOrder.splice(i, 0, name.split(':')[0]);
   } else {
-    name = './features/' + name;
+    name = './features/src/' + name;
+    if (name in dups) {
+        cpSync(
+            name,
+            name = name + "." + (++dups[name]),
+            {recursive: true}
+        );
+    } else {
+        dups[name] = 0;
+    }
     json.features[name] = feature;
     json.overrideFeatureInstallOrder.splice(i, 0, name);
   }

.gitignore

@@ -2,3 +2,4 @@
 /.cache
 /.config
 /.scratch
+/features/src/*.[0-9]

features/src/cccl-dev/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "NVIDIA CCCL development utilities",
   "id": "cccl-dev",
-  "version": "24.2.1",
+  "version": "24.2.2",
   "description": "A feature to install NVIDIA CCCL development utilities",
   "options": {
     "litVersion": {

features/src/cccl-dev/install.sh

@@ -40,21 +40,14 @@ fi
 CC=gcc CXX=g++ python -m pip install $BREAK_PACKAGES --upgrade pip
 CC=gcc CXX=g++ python -m pip install $BREAK_PACKAGES wheel setuptools;
 CC=gcc CXX=g++ python -m pip install $BREAK_PACKAGES psutil $LIT_VERSION_TO_INSTALL;
-CC=gcc CXX=g++ python -m pip install $BREAK_PACKAGES clang-format==17.*;
 
 export LIT_VERSION="$(lit --version | grep -o -e '[0-9].*')";
 
-# there must be a cleaner way, but it works
-export CLANG_FORMAT_PATH="$(pip show clang-format | grep "^Location: " | awk '{print $2}')/clang-format";
-
 # export envvars in bashrc files
 append_to_etc_bashrc "$(cat .bashrc | envsubst '$LIT_VERSION')";
-append_to_etc_bashrc "$(cat .bashrc | envsubst '$CLANG_FORMAT_PATH')";
 append_to_all_bashrcs "$(cat .bashrc | envsubst '$LIT_VERSION')";
-append_to_all_bashrcs "$(cat .bashrc | envsubst '$CLANG_FORMAT_PATH')";
 # export envvars in /etc/profile.d
 add_etc_profile_d_script cccl-dev "$(cat .bashrc | envsubst '$LIT_VERSION')";
-add_etc_profile_d_script cccl-dev "$(cat .bashrc | envsubst '$CLANG_FORMAT_PATH')";
 
 # Clean up
 # rm -rf /tmp/*;

features/src/llvm/devcontainer-feature.json

@@ -1,7 +1,7 @@
 {
   "name": "LLVM compilers and tools",
   "id": "llvm",
-  "version": "24.2.0",
+  "version": "24.2.1",
   "description": "A feature to install LLVM compilers and tools",
   "options": {
     "version": {

features/src/llvm/install.sh

@@ -40,12 +40,16 @@ echo "Installing llvm-${LLVM_VERSION} compilers and tools";
 
 ./llvm.sh $LLVM_VERSION ${PACKAGES:-all};
 
+if [[ "${PACKAGES:-all}" == "all" ]]; then
+    PACKAGES=("clang" "clangd" "clang++" "clang-format" "clang-tidy" "lldb" "llvm-config" "llvm-cov");
+else
+    PACKAGES=(${PACKAGES});
+fi
+
 # Remove existing, install, and set default clang/llvm alternatives
-for x in clang clangd clang++ clang-format clang-tidy lldb llvm-config llvm-cov; do
+for ((i=0; i < ${#PACKAGES[@]}; i+=1)); do
+    x=${PACKAGES[$i]};
     if type ${x}-${LLVM_VERSION} >/dev/null 2>&1; then
-        if ${x} != clang-format; then
-            (update-alternatives --remove-all ${x} >/dev/null 2>&1 || true);
-        fi
         (update-alternatives --install /usr/bin/${x} ${x} $(which ${x}-${LLVM_VERSION}) 30);
         (update-alternatives --set ${x} $(which ${x}-${LLVM_VERSION}));
     fi

image/.devcontainer/devcontainer.json

@@ -23,24 +23,24 @@
     },
     "ghcr.io/devcontainers/features/git-lfs:1": {},
     "ghcr.io/devcontainers/features/github-cli:1": {},
-    "./features/gitlab-cli": {},
-    "./features/cmake": {},
-    "./features/ninja": {},
-    "./features/sccache": {
+    "./features/src/gitlab-cli": {},
+    "./features/src/cmake": {},
+    "./features/src/ninja": {},
+    "./features/src/sccache": {
       "version": "0.7.4"
     },
-    "./features/utils": {}
+    "./features/src/utils": {}
   },
   "overrideFeatureInstallOrder": [
     "ghcr.io/devcontainers/features/common-utils",
     "ghcr.io/devcontainers/features/git",
     "ghcr.io/devcontainers/features/git-lfs",
     "ghcr.io/devcontainers/features/github-cli",
-    "./features/gitlab-cli",
-    "./features/cmake",
-    "./features/ninja",
-    "./features/sccache",
-    "./features/utils"
+    "./features/src/gitlab-cli",
+    "./features/src/cmake",
+    "./features/src/ninja",
+    "./features/src/sccache",
+    "./features/src/utils"
   ],
   "updateContentCommand": [
     "/bin/bash",

image/.devcontainer/features

@@ -1 +1 @@
-../../features/src
+../../features

matrix.yml

@@ -36,6 +36,7 @@ x-mambaforge: &conda { name: "mambaforge" }
 
 x-python: &python { name: "ghcr.io/devcontainers/features/python:1", version: "os-provided", installTools: "false", hide: true }
 x-cccl_dev: &cccl_dev { name: "cccl-dev", hide: true }
+x-cccl-clang-tools: &cccl_clang_tools {name: "llvm", version: "17", packages: "clang-format clangd", hide: true}
 
 # CCCL only needs a subset of the full CTK:
 x-cccl-cuda-opts: &cccl_cuda_opts {
@@ -52,46 +53,46 @@ include:
 
 - os: "ubuntu:18.04"
   images:
-  - {features: [*gcc_6, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_7, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_8, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_9, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*llvm_9, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*oneapi_2022, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *onapi_env}
+  - {features: [*gcc_6, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_7, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_8, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_9, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*llvm_9, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*oneapi_2022, {<<: *cuda_prev_min, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *onapi_env}
 
 - os: "ubuntu:20.04"
   images:
-  - {features: [*gcc_7, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_8, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_9, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_10, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*llvm_9, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*llvm_10, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*llvm_11, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*llvm_12, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*llvm_13, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*llvm_14, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*oneapi_2022, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *onapi_env}
+  - {features: [*gcc_7, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_8, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_9, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_10, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*llvm_9, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*llvm_10, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*llvm_11, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*llvm_12, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*llvm_13, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*llvm_14, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*oneapi_2022, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *onapi_env}
 
 - os: "ubuntu:22.04"
   images:
-  - {features: [*nvhpc_prev, *python, *cccl_dev], env: *nvhpc_env}
-  - {features: [*nvhpc_curr, *python, *cccl_dev], env: *nvhpc_env}
-  - {features: [*gcc_11, {<<: *cuda_prev_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_11, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_12, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*gcc_13, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
-  - {features: [*llvm_15, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*llvm_prev, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*llvm_curr, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *llvm_env}
-  - {features: [*oneapi_2022, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *onapi_env}
+  - {features: [*nvhpc_prev, *python, *cccl_clang_tools, *cccl_dev], env: *nvhpc_env}
+  - {features: [*nvhpc_curr, *python, *cccl_clang_tools, *cccl_dev], env: *nvhpc_env}
+  - {features: [*gcc_11, {<<: *cuda_prev_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_11, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_12, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_13, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
+  - {features: [*llvm_15, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*llvm_prev, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*llvm_curr, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *llvm_env}
+  - {features: [*oneapi_2022, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *onapi_env}
   # llvm-cuda-nvhpc (needed by stdexec devcontainers)
-  - {features: [*llvm_curr, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *nvhpc_curr, *python]}
+  - {features: [*llvm_curr, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *nvhpc_curr, *python, *cccl_clang_tools]}
 
 # Hack for getting gcc-13.2
 - os: "ubuntu:23.04"
   images:
-  - {features: [*gcc_13, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_dev], env: *gcc_env}
+  - {features: [*gcc_13, {<<: *cuda_curr_max, <<: *cccl_cuda_opts}, *python, *cccl_clang_tools, *cccl_dev], env: *gcc_env}
 
 - os: "windows"
   images:

scripts/build.sh

@@ -12,9 +12,17 @@ build_devcontainer() {
     # Generate the devcontainer
     eval "$(./scripts/generate.sh "$@" | xargs -r -d'\n' -I% echo -n local %\;)";
 
+    trap "$(cat <<________
+        code=\$?;
+        find ./features/src -maxdepth 1 -type d -name '*\.[0-9]' -exec rm -r "{}" \;
+        exit \$code;
+________
+)" EXIT;
+
     # Build the devcontainer
-    exec devcontainer build \
+    devcontainer build \
         --workspace-folder "${workspace}" \
+        --cache-from "docker.io/rapidsai/devcontainers:${tag}" \
         --image-name "docker.io/rapidsai/devcontainers:${tag}" \
         ;
 }

scripts/generate.sh

@@ -14,7 +14,7 @@ generate_devcontainer() {
     rm -rf "${workspace}";
     cp -ar image "${workspace}";
     rm "${workspace}/.devcontainer/features";
-    ln -s "$(realpath ./features/src)" "${workspace}/.devcontainer/features";
+    ln -s "$(realpath ./features)" "${workspace}/.devcontainer/features";
 
     # Generate tag and devcontainer.json
     .github/actions/devcontainer-json/action.sh           \

scripts/run.sh

@@ -13,6 +13,14 @@ run_devcontainer() {
     eval "$(./scripts/generate.sh "$@" | xargs -r -d'\n' -I% echo -n local %\;)";
 
     # Build and tag the devcontainer
+
+    trap "$(cat <<________
+        code=\$?;
+        find ./features/src -maxdepth 1 -type d -name '*\.[0-9]' -exec rm -r "{}" \;
+        exit \$code;
+________
+)" EXIT;
+
     devcontainer build \
         --workspace-folder "${workspace}" \
         --cache-from "docker.io/rapidsai/devcontainers:${tag}" \
@@ -57,7 +65,7 @@ run_devcontainer() {
         --mount "type=bind,source=$(pwd)/.scratch/.cache,target=/home/coder/.cache" \
         --mount "type=bind,source=$(pwd)/.scratch/.config,target=/home/coder/.config" \
         --mount "type=bind,source=$(pwd)/features/src/utils/opt/devcontainer,target=/opt/devcontainer" \
-        --additional-features '{ "./features/rapids-build-utils": {} }' \
+        --additional-features '{ "./features/src/rapids-build-utils": {} }' \
         --mount "type=bind,source=$(pwd)/features/src/rapids-build-utils/opt/rapids-build-utils,target=/opt/rapids-build-utils" \
         ;
 
@@ -71,6 +79,7 @@ run_devcontainer() {
         docker rm -f '${container_id}' >/dev/null 2>&1 || true;
         docker rmi -f '${image_tag}' >/dev/null 2>&1 || true;
         docker rmi -f '${image_tag%%-uid*}' >/dev/null 2>&1 || true;
+        find ./features/src -maxdepth 1 -type d -name '*\.[0-9]' -exec rm -r "{}" \;
         exit \$code;
 ________
 )" EXIT;