From e93632ba266a881ef286e18f1ba4a263304da6b2 Mon Sep 17 00:00:00 2001
From: Yuuichi Asahi <y.asahi@nr.titech.ac.jp>
Date: Fri, 19 Jan 2024 21:24:34 +0900
Subject: [PATCH] fix kokkos path and add install tests on cuda and hip

---
 .github/workflows/cmake.yml      | 18 +++++++--
 install_test/bin/install_cpu.sh  |  4 +-
 install_test/bin/install_cuda.sh | 68 ++++++++++++++++++++++++++++++++
 install_test/bin/install_hip.sh  | 68 ++++++++++++++++++++++++++++++++
 4 files changed, 153 insertions(+), 5 deletions(-)
 create mode 100755 install_test/bin/install_cuda.sh
 create mode 100755 install_test/bin/install_hip.sh

diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml
index b9a92f8d..2755d1ce 100644
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@@ -80,15 +80,23 @@ jobs:
           docker run -v ${{github.workspace}}:/work ${{ env.container }} cmake --build build_${backend} --config ${{env.BUILD_TYPE}} -j 2
         done
 
-    - name: Test on CPUs
+    - name: Test for OpenMP backend
       # Execute tests defined by the CMake configuration. Testing on CPUs only
       run: |
         docker run -v ${{github.workspace}}:/work ${{ env.container }} ctest --output-on-failure --test-dir build_OPENMP -C ${{env.BUILD_TYPE}}
 
-    - name: Install test on CPUs
+    - name: Install test for OpenMP backend
       run: |
         docker run -v ${{github.workspace}}:/work ${{ env.container }} ./install_test/bin/install_cpu.sh /tmp
 
+    - name: Install test for CUDA backend
+      run: |
+        for backend in ${{ env.backends }}; do
+          if [ ${backend} != "OPENMP" ]; then
+            docker run -v ${{github.workspace}}:/work ${{ env.container }} ./install_test/bin/install_cuda.sh /tmp ${backend}
+          fi
+        done
+
   build_amd:
     # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac.
     # You can convert this to a matrix build if you need cross-platform coverage.
@@ -132,4 +140,8 @@ jobs:
     - name: Build
       # Build your program with the given configuration
       run: |
-        docker run -v ${{github.workspace}}:/work ${{ env.container }} cmake --build build_${{matrix.backend.name}} --config ${{env.BUILD_TYPE}} -j 2
\ No newline at end of file
+        docker run -v ${{github.workspace}}:/work ${{ env.container }} cmake --build build_${{matrix.backend.name}} --config ${{env.BUILD_TYPE}} -j 2
+
+    - name: Install test for HIP backend
+      run: |
+        docker run -v ${{github.workspace}}:/work ${{ env.container }} ./install_test/bin/install_hip.sh /tmp ${{matrix.backend.name}}
\ No newline at end of file
diff --git a/install_test/bin/install_cpu.sh b/install_test/bin/install_cpu.sh
index b02d0b65..f0e4e4bc 100755
--- a/install_test/bin/install_cpu.sh
+++ b/install_test/bin/install_cpu.sh
@@ -7,11 +7,11 @@ TARGET="cpu"
 
 # Install Kokkos
 export KOKKOS_INSTALL_PREFIX=${ROOT_DIR}/usr/local/kokkos_${TARGET}
-export Kokkos_DIR=${KOKKOS_INSTALL_PREFIX}/lib64/cmake/Kokkos
+export Kokkos_DIR=${KOKKOS_INSTALL_PREFIX}/lib/cmake/Kokkos
 export KOKKOS_BUILD_DIR=build_Kokkos_${TARGET}
 
 export KOKKOSFFT_INSTALL_PREFIX=${ROOT_DIR}/usr/local/kokkosFFT_${TARGET}
-export KokkosFFT_DIR=${KOKKOSFFT_INSTALL_PREFIX}/lib64/cmake/kokkos-fft
+export KokkosFFT_DIR=${KOKKOSFFT_INSTALL_PREFIX}/lib/cmake/kokkos-fft
 export KOKKOSFFT_BUILD_DIR=build_KokkosFFT_${TARGET}
 
 export EXAMPLE_BUILD_DIR=build_example_${TARGET}
diff --git a/install_test/bin/install_cuda.sh b/install_test/bin/install_cuda.sh
new file mode 100755
index 00000000..43148a8c
--- /dev/null
+++ b/install_test/bin/install_cuda.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+args=$#
+ROOT_DIR=$1
+WK_DIR=$(pwd)
+TARGET=$2
+KOKKOS_TARGET="CUDA"
+
+# Install Kokkos
+export KOKKOS_INSTALL_PREFIX=${ROOT_DIR}/usr/local/kokkos_${KOKKOS_TARGET}
+export Kokkos_DIR=${KOKKOS_INSTALL_PREFIX}/lib/cmake/Kokkos
+export KOKKOS_BUILD_DIR=build_Kokkos_${KOKKOS_TARGET}
+
+export KOKKOSFFT_INSTALL_PREFIX=${ROOT_DIR}/usr/local/kokkosFFT_${TARGET}
+export KokkosFFT_DIR=${KOKKOSFFT_INSTALL_PREFIX}/lib/cmake/kokkos-fft
+export KOKKOSFFT_BUILD_DIR=build_KokkosFFT_${TARGET}
+
+export EXAMPLE_BUILD_DIR=build_example_${TARGET}
+
+# Install Kokkos
+cd ${WK_DIR}
+mkdir ${KOKKOS_BUILD_DIR} && cd ${KOKKOS_BUILD_DIR}
+
+# Get Kokkos from github repo and build
+git clone https://github.com/kokkos/kokkos.git
+cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+      -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON \
+      -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} kokkos
+
+cmake --build . -j 8
+cmake --install .
+
+# Install KokkosFFT
+cd ${WK_DIR}
+mkdir ${KOKKOSFFT_BUILD_DIR} && cd ${KOKKOSFFT_BUILD_DIR}
+if [ $TARGET == "CUDA" ]; then
+    cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+            -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON \
+            -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} ..
+else
+    cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON \
+          -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON ..
+fi
+cmake --build . -j 8
+cmake --install .
+
+# Try to build an example
+# Build KokkosFFT code using installed KokkosFFT
+cd ${WK_DIR}
+mkdir ${EXAMPLE_BUILD_DIR} && cd ${EXAMPLE_BUILD_DIR}
+if [ $TARGET == "CUDA" ]; then
+    cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON \
+          -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} ../install_test/src
+else
+    cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON \
+          -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON ../install_test/src
+fi
+cmake --build . -j 8
+
+if [ $? -eq 0 ]; then
+    echo "*** install test: build SUCCESSFUL ***"
+else
+    echo "*** install test: build FAILED ***"
+    exit 1;
+fi
\ No newline at end of file
diff --git a/install_test/bin/install_hip.sh b/install_test/bin/install_hip.sh
new file mode 100755
index 00000000..1b64f38a
--- /dev/null
+++ b/install_test/bin/install_hip.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+args=$#
+ROOT_DIR=$1
+WK_DIR=$(pwd)
+TARGET=$2
+KOKKOS_TARGET="HIP"
+
+# Install Kokkos
+export KOKKOS_INSTALL_PREFIX=${ROOT_DIR}/usr/local/kokkos_${KOKKOS_TARGET}
+export Kokkos_DIR=${KOKKOS_INSTALL_PREFIX}/lib/cmake/Kokkos
+export KOKKOS_BUILD_DIR=build_Kokkos_${KOKKOS_TARGET}
+
+export KOKKOSFFT_INSTALL_PREFIX=${ROOT_DIR}/usr/local/kokkosFFT_${TARGET}
+export KokkosFFT_DIR=${KOKKOSFFT_INSTALL_PREFIX}/lib/cmake/kokkos-fft
+export KOKKOSFFT_BUILD_DIR=build_KokkosFFT_${TARGET}
+
+export EXAMPLE_BUILD_DIR=build_example_${TARGET}
+
+# Install Kokkos
+cd ${WK_DIR}
+mkdir ${KOKKOS_BUILD_DIR} && cd ${KOKKOS_BUILD_DIR}
+
+# Get Kokkos from github repo and build
+git clone https://github.com/kokkos/kokkos.git
+cmake -DCMAKE_CXX_COMPILER=hipcc \
+      -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON \
+      -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} kokkos
+
+cmake --build . -j 8
+cmake --install .
+
+# Install KokkosFFT
+cd ${WK_DIR}
+mkdir ${KOKKOSFFT_BUILD_DIR} && cd ${KOKKOSFFT_BUILD_DIR}
+if [ $TARGET == "HIP" ]; then
+    cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VEGA90A=ON \
+          -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} ..
+else
+    cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VEGA90A=ON \
+          -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON ..
+fi
+cmake --build . -j 8
+cmake --install .
+
+# Try to build an example
+# Build KokkosFFT code using installed KokkosFFT
+cd ${WK_DIR}
+mkdir ${EXAMPLE_BUILD_DIR} && cd ${EXAMPLE_BUILD_DIR}
+if [ $TARGET == "HIP" ]; then
+    cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VEGA90A=ON \
+          -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} ../install_test/src
+else
+    cmake -DCMAKE_CXX_COMPILER=kokkos/bin/nvcc_wrapper \
+          -DCMAKE_CXX_STANDARD=17 -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VEGA90A=ON \
+          -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PREFIX} -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON ../install_test/src
+fi
+cmake --build . -j 8
+
+if [ $? -eq 0 ]; then
+    echo "*** install test: build SUCCESSFUL ***"
+else
+    echo "*** install test: build FAILED ***"
+    exit 1;
+fi
\ No newline at end of file