expand rapids devcontainer matrix (#111)

* add matrix of RAPIDS devcontainers * remove unnecessary tag * install common clangd config into /etc/skel * try to auth with lower and uppercase org names in the vault URL
rapidsai · Aug 17, 2023 · 594c979 · 594c979
1 parent 7a1f551
commit 594c979
Show file tree

Hide file tree

Showing 58 changed files with 1,137 additions and 61 deletions.
diff --git a/.devcontainer/cuda11.8-conda/devcontainer.json b/.devcontainer/cuda11.8-conda/devcontainer.json
@@ -0,0 +1,100 @@
+{
+  "shutdownAction": "stopContainer",
+
+  "build": {
+    "context": "${localWorkspaceFolder}/.devcontainer",
+    "dockerfile": "${localWorkspaceFolder}/.devcontainer/rapids.Dockerfile",
+    "args": {
+      "CUDA": "11.8",
+      "PYTHON_PACKAGE_MANAGER": "conda",
+      "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-mambaforge-ubuntu22.04"
+    }
+  },
+  "hostRequirements": {
+    "gpu": true
+  },
+
+  "features": {
+    "./features/rapids-build-utils": {}
+  },
+
+  "overrideFeatureInstallOrder": [
+    "./features/rapids-build-utils"
+  ],
+
+  "initializeCommand": [
+    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config,conda/pkgs,conda/unified} ${localWorkspaceFolder}/../{rmm,kvikio,cudf,raft,cumlprims_mg,cuml,cugraph-ops,cugraph,cuspatial}"
+  ],
+
+  "updateContentCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -m 0755 -p ~/.config/clangd && cp -n /etc/skel/.config/clangd/config.yaml ~/.config/clangd/config.yaml"
+  ],
+
+  "workspaceFolder": "/home/coder",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent",
+  "mounts": [
+    "source=${localWorkspaceFolder}/../rmm,target=/home/coder/rmm,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../kvikio,target=/home/coder/kvikio,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cudf,target=/home/coder/cudf,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../raft,target=/home/coder/raft,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cumlprims_mg,target=/home/coder/cumlprims_mg,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cuml,target=/home/coder/cuml,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cugraph-ops,target=/home/coder/cugraph-ops,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cugraph,target=/home/coder/cugraph,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cuspatial,target=/home/coder/cuspatial,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.conda/pkgs,target=/home/coder/.conda/pkgs,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.conda/unified,target=/home/coder/.conda/envs,type=bind,consistency=consistent"
+  ],
+
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "llvm-vs-code-extensions.vscode-clangd",
+        "mutantdino.resourcemonitor",
+        "ms-vscode.cpptools",
+        "nvidia.nsight-vscode-edition",
+        "seaube.clangformat",
+        "tamasfe.even-better-toml"
+      ],
+      "settings": {
+        "C_Cpp.vcpkg.enabled": false,
+        "C_Cpp.formatting": "disabled",
+        "C_Cpp.autocomplete": "disabled",
+        "C_Cpp.errorSquiggles": "disabled",
+        "C_Cpp.intelliSenseEngine": "disabled",
+        "C_Cpp.configurationWarnings": "disabled",
+        "C_Cpp.autoAddFileAssociations": false,
+        "clang-format.fallbackStyle": "none",
+        "files.trimFinalNewlines": true,
+        "files.insertFinalNewline": true,
+        "files.trimTrailingWhitespace": true,
+        "files.associations": {
+          "*.cu": "cuda-cpp",
+          "*.cuh": "cuda-cpp",
+          "**/libcudacxx/include/**/*": "cpp",
+          "**/libcudacxx-src/include/**/*": "cpp"
+        },
+        "files.watcherExclude": {
+          "**/build/**": true,
+          "**/_skbuild/**": true,
+          "**/target/**": true
+        },
+        "python.linting.flake8Enabled": true,
+        "[c]": {
+          "editor.defaultFormatter": "seaube.clangformat"
+        },
+        "[cpp]": {
+          "editor.defaultFormatter": "seaube.clangformat"
+        },
+        "[cuda-cpp]": {
+          "editor.defaultFormatter": "seaube.clangformat"
+        }
+      }
+    }
+  }
+}
diff --git a/.devcontainer/conda/features → .devcontainer/cuda11.8-conda/features b/.devcontainer/conda/features → .devcontainer/cuda11.8-conda/features
diff --git a/.devcontainer/cuda11.8-pip/devcontainer.json b/.devcontainer/cuda11.8-pip/devcontainer.json
@@ -0,0 +1,99 @@
+{
+  "shutdownAction": "stopContainer",
+
+  "build": {
+    "context": "${localWorkspaceFolder}/.devcontainer",
+    "dockerfile": "${localWorkspaceFolder}/.devcontainer/rapids.Dockerfile",
+    "args": {
+      "CUDA": "11.8",
+      "PYTHON_PACKAGE_MANAGER": "pip",
+      "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda11.8-ubuntu22.04"
+    }
+  },
+  "hostRequirements": {
+    "gpu": true
+  },
+
+  "features": {
+    "./features/rapids-build-utils": {}
+  },
+
+  "overrideFeatureInstallOrder": [
+    "./features/rapids-build-utils"
+  ],
+
+  "initializeCommand": [
+    "/bin/bash", "-c", "mkdir -m 0755 -p ${localWorkspaceFolder}/../.{aws,cache,config/pip,local/unified} ${localWorkspaceFolder}/../{rmm,kvikio,cudf,raft,cumlprims_mg,cuml,cugraph-ops,cugraph,cuspatial}"
+  ],
+
+  "updateContentCommand": [
+    "/bin/bash",
+    "-c",
+    "mkdir -m 0755 -p ~/.config/clangd && cp -n /etc/skel/.config/clangd/config.yaml ~/.config/clangd/config.yaml"
+  ],
+
+  "workspaceFolder": "/home/coder",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent",
+  "mounts": [
+    "source=${localWorkspaceFolder}/../rmm,target=/home/coder/rmm,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../kvikio,target=/home/coder/kvikio,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cudf,target=/home/coder/cudf,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../raft,target=/home/coder/raft,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cumlprims_mg,target=/home/coder/cumlprims_mg,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cuml,target=/home/coder/cuml,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cugraph-ops,target=/home/coder/cugraph-ops,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cugraph,target=/home/coder/cugraph,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../cuspatial,target=/home/coder/cuspatial,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.aws,target=/home/coder/.aws,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.cache,target=/home/coder/.cache,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.config,target=/home/coder/.config,type=bind,consistency=consistent",
+    "source=${localWorkspaceFolder}/../.local/unified,target=/home/coder/.local,type=bind,consistency=consistent"
+  ],
+
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "llvm-vs-code-extensions.vscode-clangd",
+        "mutantdino.resourcemonitor",
+        "ms-vscode.cpptools",
+        "nvidia.nsight-vscode-edition",
+        "seaube.clangformat",
+        "tamasfe.even-better-toml"
+      ],
+      "settings": {
+        "C_Cpp.vcpkg.enabled": false,
+        "C_Cpp.formatting": "disabled",
+        "C_Cpp.autocomplete": "disabled",
+        "C_Cpp.errorSquiggles": "disabled",
+        "C_Cpp.intelliSenseEngine": "disabled",
+        "C_Cpp.configurationWarnings": "disabled",
+        "C_Cpp.autoAddFileAssociations": false,
+        "clang-format.fallbackStyle": "none",
+        "files.trimFinalNewlines": true,
+        "files.insertFinalNewline": true,
+        "files.trimTrailingWhitespace": true,
+        "files.associations": {
+          "*.cu": "cuda-cpp",
+          "*.cuh": "cuda-cpp",
+          "**/libcudacxx/include/**/*": "cpp",
+          "**/libcudacxx-src/include/**/*": "cpp"
+        },
+        "files.watcherExclude": {
+          "**/build/**": true,
+          "**/_skbuild/**": true,
+          "**/target/**": true
+        },
+        "python.linting.flake8Enabled": true,
+        "[c]": {
+          "editor.defaultFormatter": "seaube.clangformat"
+        },
+        "[cpp]": {
+          "editor.defaultFormatter": "seaube.clangformat"
+        },
+        "[cuda-cpp]": {
+          "editor.defaultFormatter": "seaube.clangformat"
+        }
+      }
+    }
+  }
+}
diff --git a/.devcontainer/pip/features → .devcontainer/cuda11.8-pip/features b/.devcontainer/pip/features → .devcontainer/cuda11.8-pip/features
diff --git a/.devcontainer/conda/devcontainer.json → ...ontainer/cuda12.0-conda/devcontainer.json b/.devcontainer/conda/devcontainer.json → ...ontainer/cuda12.0-conda/devcontainer.json
@@ -6,8 +6,8 @@
     "dockerfile": "${localWorkspaceFolder}/.devcontainer/rapids.Dockerfile",
     "args": {
       "CUDA": "12.0",
-      "LLVM": "16",
-      "PYTHON_PACKAGE_MANAGER": "conda"
+      "PYTHON_PACKAGE_MANAGER": "conda",
+      "BASE": "rapidsai/devcontainers:23.10-cpp-mambaforge-ubuntu22.04"
     }
   },
   "hostRequirements": {
@@ -32,12 +32,8 @@
     "mkdir -m 0755 -p ~/.config/clangd && cp -n /etc/skel/.config/clangd/config.yaml ~/.config/clangd/config.yaml"
   ],
 
-  "containerEnv": {
-    "DEFAULT_CONDA_ENV": "rapids"
-  },
-
   "workspaceFolder": "/home/coder",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent",
   "mounts": [
     "source=${localWorkspaceFolder}/../rmm,target=/home/coder/rmm,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../kvikio,target=/home/coder/kvikio,type=bind,consistency=consistent",

diff --git a/.devcontainer/cuda12.0-conda/features b/.devcontainer/cuda12.0-conda/features
@@ -0,0 +1 @@
+../../features/src
diff --git a/.devcontainer/pip/devcontainer.json → .devcontainer/cuda12.0-pip/devcontainer.json b/.devcontainer/pip/devcontainer.json → .devcontainer/cuda12.0-pip/devcontainer.json
@@ -6,8 +6,8 @@
     "dockerfile": "${localWorkspaceFolder}/.devcontainer/rapids.Dockerfile",
     "args": {
       "CUDA": "12.0",
-      "LLVM": "16",
-      "PYTHON_PACKAGE_MANAGER": "pip"
+      "PYTHON_PACKAGE_MANAGER": "pip",
+      "BASE": "rapidsai/devcontainers:23.10-cpp-llvm16-cuda12.0-ubuntu22.04"
     }
   },
   "hostRequirements": {
@@ -32,12 +32,8 @@
     "mkdir -m 0755 -p ~/.config/clangd && cp -n /etc/skel/.config/clangd/config.yaml ~/.config/clangd/config.yaml"
   ],
 
-  "containerEnv": {
-    "DEFAULT_VIRTUAL_ENV": "rapids"
-  },
-
   "workspaceFolder": "/home/coder",
-  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/${localWorkspaceFolderBasename},type=bind,consistency=consistent",
+  "workspaceMount": "source=${localWorkspaceFolder},target=/home/coder/devcontainers,type=bind,consistency=consistent",
   "mounts": [
     "source=${localWorkspaceFolder}/../rmm,target=/home/coder/rmm,type=bind,consistency=consistent",
     "source=${localWorkspaceFolder}/../kvikio,target=/home/coder/kvikio,type=bind,consistency=consistent",

diff --git a/.devcontainer/cuda12.0-pip/features b/.devcontainer/cuda12.0-pip/features
@@ -0,0 +1 @@
+../../features/src
diff --git a/.devcontainer/rapids.Dockerfile b/.devcontainer/rapids.Dockerfile
@@ -1,18 +1,15 @@
 # syntax=docker/dockerfile:1.5
 
-ARG CUDA=12.0
-ARG LLVM=16
-ARG RAPIDS=23.10
-ARG DISTRO=ubuntu22.04
-ARG REPO=rapidsai/devcontainers
-
+ARG BASE
 ARG PYTHON_PACKAGE_MANAGER=conda
 
-FROM ${REPO}:${RAPIDS}-cpp-llvm${LLVM}-cuda${CUDA}-${DISTRO} as pip-base
+FROM ${BASE} as pip-base
+
+ENV DEFAULT_VIRTUAL_ENV=rapids
 
-FROM ${REPO}:${RAPIDS}-cpp-mambaforge-${DISTRO} as conda-base
+FROM ${BASE} as conda-base
 
-COPY --from=pip-base /etc/skel/.config/clangd/config.yaml /etc/skel/.config/clangd/config.yaml
+ENV DEFAULT_CONDA_ENV=rapids
 
 FROM ${PYTHON_PACKAGE_MANAGER}-base
 

diff --git a/.github/workflows/build-and-push-image.yml b/.github/workflows/build-and-push-image.yml
@@ -95,11 +95,9 @@ jobs:
           hashes=();
 
           for arch in "amd64" "arm64"; do
-            # Load the linux/$arch and tag the loaded image with the final name
-            docker image tag \
-              $(docker load --input "${{ runner.temp }}/${arch}.tar" | tail -n1 | cut -d' ' -f3) \
-              ${name};
-            # Push the local image to the registry to get its remote hash and save the remote hash for use below
+            # Load the arch-specific image with the final name
+            docker load --input "${{ runner.temp }}/${arch}.tar";
+            # Push the local image to the registry and save the remote hash
             hashes+=("$(docker push ${name} | tail -n1 | cut -d' ' -f3)");
           done
 

diff --git a/features/src/llvm/.clangd → ...ommon/etc/skel/.config/clangd/config.yaml b/features/src/llvm/.clangd → ...ommon/etc/skel/.config/clangd/config.yaml
diff --git a/features/common/install.sh b/features/common/install.sh
@@ -11,6 +11,8 @@ src="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # export bash utility functions
 source "$src/utilities.sh";
 
+# install /etc/skel
+cp -r "$src/etc/skel" /etc/;
 # install /etc/bash.bash_env
 cp "$src/etc/bash.bash_env" /etc/;
 chown root:root /etc/bash.bash_env;

diff --git a/features/src/cmake/common/etc/skel/.config/clangd/config.yaml b/features/src/cmake/common/etc/skel/.config/clangd/config.yaml
@@ -0,0 +1,61 @@
+# https://clangd.llvm.org/config
+
+# Apply a config conditionally to all C files
+If:
+  PathMatch: .*\.(c|h)$
+
+---
+
+# Apply a config conditionally to all C++ files
+If:
+  PathMatch: .*\.(c|h)pp
+
+---
+
+# Apply a config conditionally to all CUDA files
+If:
+  PathMatch: .*\.cuh?
+CompileFlags:
+  Add:
+    - "-x"
+    - "cuda"
+    # No error on unknown CUDA versions
+    - "-Wno-unknown-cuda-version"
+    # Allow variadic CUDA functions
+    - "-Xclang=-fcuda-allow-variadic-functions"
+
+---
+
+# Tweak the clangd parse settings for all files
+CompileFlags:
+  Add:
+    # report all errors
+    - "-ferror-limit=0"
+    - "-fmacro-backtrace-limit=0"
+    - "-ftemplate-backtrace-limit=0"
+    # Skip the CUDA version check
+    - "--no-cuda-version-check"
+  Remove:
+    # remove gcc's -fcoroutines
+    - -fcoroutines
+    # remove nvc++ flags unknown to clang
+    - "-gpu=*"
+    - "-stdpar*"
+    # remove nvcc flags unknown to clang
+    - "-arch*"
+    - "-gencode*"
+    - "--generate-code*"
+    - "-ccbin*"
+    - "-t=*"
+    - "--threads*"
+    - "-Xptxas*"
+    - "-Xcudafe*"
+    - "-Xfatbin*"
+    - "-Xcompiler*"
+    - "--diag-suppress*"
+    - "--diag_suppress*"
+    - "--compiler-options*"
+    - "--expt-extended-lambda"
+    - "--expt-relaxed-constexpr"
+    - "-forward-unknown-to-host-compiler"
+    - "-Werror=cross-execution-space-call"
diff --git a/features/src/cmake/common/install.sh b/features/src/cmake/common/install.sh
@@ -11,6 +11,8 @@ src="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # export bash utility functions
 source "$src/utilities.sh";
 
+# install /etc/skel
+cp -r "$src/etc/skel" /etc/;
 # install /etc/bash.bash_env
 cp "$src/etc/bash.bash_env" /etc/;
 chown root:root /etc/bash.bash_env;

diff --git a/features/src/cmake/devcontainer-feature.json b/features/src/cmake/devcontainer-feature.json
@@ -1,7 +1,7 @@
 {
   "name": "CMake",
   "id": "cmake",
-  "version": "23.10.0",
+  "version": "23.10.1",
   "description": "A feature to install CMake",
   "options": {
     "version": {