diff --git a/.jenkins/continuous.groovy b/.jenkins/continuous.groovy index 1e2283c7..f34dd155 100644 --- a/.jenkins/continuous.groovy +++ b/.jenkins/continuous.groovy @@ -42,6 +42,28 @@ pipeline { make -j8 && cd unit_tests && mpirun -np 2 ./KokkosRemoteSpaces_TestAll''' } } + stage('nvshmem') { + agent { + dockerfile { + filename 'Dockerfile.nvshmem' + dir 'scripts/docker' + label 'nvidia-docker && large_images' + } + } + steps { + sh '''rm -rf build && mkdir -p build && cd build && \ + cmake \ + -DCMAKE_BUILD_TYPE=Release \ + -DKokkos_DIR=${KOKKOS_ROOT} \ + -DCMAKE_CXX_COMPILER=${KOKKOS_ROOT}/bin/nvcc_wrapper \ + -DKRS_ENABLE_NVSHMEMSPACE=ON \ + -DNVSHMEM_ROOT=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/comm_libs/nvshmem \ + -DKRS_ENABLE_TESTS=ON \ + -DCMAKE_CXX_FLAGS=-Werror \ + .. && \ + make -j8 && cd unit_tests && mpirun -np 2 ./KokkosRemoteSpaces_TestAll''' + } + } } } } diff --git a/scripts/docker/Dockerfile.nvshmem b/scripts/docker/Dockerfile.nvshmem new file mode 100644 index 00000000..d3363e8c --- /dev/null +++ b/scripts/docker/Dockerfile.nvshmem @@ -0,0 +1,66 @@ +FROM nvcr.io/nvidia/nvhpc:23.7-devel-cuda12.2-ubuntu20.04 + +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get upgrade -y && apt-get install -y \ + build-essential \ + wget \ + git \ + bc \ + ninja-build \ + git \ + libev-dev \ + libevent-dev \ + libhwloc-dev \ + pkg-config \ + clang-format-8 \ + && \ + apt-get clean && rm -rf /var/lib/apt/list + +ENV PREFIX=/scratch +ENV ARCHIVE_DIR=${PREFIX}/archive +ENV SOURCE_DIR=${PREFIX}/source +ENV BUILD_DIR=${PREFIX}/build +ENV INSTALL_DIR=/opt + +RUN mkdir -p ${PREFIX} && \ + cd ${PREFIX} && \ + mkdir archive && \ + mkdir source && \ + mkdir build + +# Install CMake +RUN export CMAKE_VERSION=3.22.2 && \ + export CMAKE_SHA256=38b3befdee8fd2bac06954e2a77cb3072e6833c69d8cc013c0a3b26f1cfdfe37 && \ + export CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && \ + export CMAKE_ARCHIVE=${ARCHIVE_DIR}/cmake.tar.gz && \ + export CMAKE_BUILD_DIR=${BUILD_DIR}/cmake && \ + wget --quiet ${CMAKE_URL} --output-document=${CMAKE_ARCHIVE} && \ + echo "${CMAKE_SHA256} ${CMAKE_ARCHIVE}" | sha256sum -c && \ + mkdir -p ${CMAKE_BUILD_DIR} && \ + tar xf ${CMAKE_ARCHIVE} -C ${CMAKE_BUILD_DIR} --strip-components=1 && \ + mv ${CMAKE_BUILD_DIR} ${INSTALL_DIR} && \ + rm -rf ${CMAKE_ARCHIVE} && \ + rm -rf ${CMAKE_BUILD_DIR} +ENV PATH=${INSTALL_DIR}/cmake/bin:$PATH + +## Install Kokkos +RUN export KOKKOS_SOURCE_DIR=${SOURCE_DIR}/kokkos && \ + export KOKKOS_BUILD_DIR=${BUILD_DIR}/kokkos && \ + export KOKKOS_INSTALL_DIR=${INSTALL_DIR}/kokkos && \ + cd ${SOURCE_DIR} && git clone https://github.com/kokkos/kokkos && \ + cd kokkos && \ + git checkout 4.1.00 && \ + mkdir -p ${KOKKOS_BUILD_DIR} && \ + cd ${KOKKOS_BUILD_DIR} && \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=${KOKKOS_SOURCE_DIR}/bin/nvcc_wrapper \ + -DBUILD_SHARED_LIBS=ON \ + -DCUDAToolkit_ROOT=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/cuda \ + -DKokkos_ENABLE_CUDA=ON \ + -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_DIR} \ + ${KOKKOS_SOURCE_DIR} && \ + make -j${N_PROCS} install && \ + rm -rf ${KOKKOS_ARCHIVE} && \ + rm -rf ${KOKKOS_BUILD_DIR} && \ + rm -rf ${KOKKOS_SOURCE_DIR} +ENV KOKKOS_ROOT=${INSTALL_DIR}/kokkos