cmake.yml

name: Build and test

on:
  pull_request:
    branches:
      - main

env:
  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
  BUILD_TYPE: RelWithDebInfo

  # Force the use of BuildKit for Docker
  DOCKER_BUILDKIT: 1

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - uses: DoozyX/clang-format-lint-action@v0.13
      with:
        source: 'common/ fft/ examples/'
        exclude: ''
        extensions: 'hpp,cpp'
        clangFormatVersion: 12

  check_docker_files:
    runs-on: ubuntu-latest

    outputs:
      # true if any Docker file was modified in the PR (PR mode) or since last pushed commit (push mode)
      docker_files_have_changed: ${{ steps.get_changed_docker_files.outputs.any_changed == 'true' }}
      # use "_pr" as image name suffix if on PR mode and if any Docker file was modified
      image_name_suffix: ${{ steps.get_changed_docker_files.outputs.any_changed == 'true' && github.event_name == 'pull_request' && '_pr' || '' }}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Get changed Dockerfiles
        id: get_changed_docker_files
        uses: tj-actions/changed-files@v42
        with:
          files: docker/**/Dockerfile

      - name: List changed Dockerfiles
        if: ${{ steps.docker_files_have_changed.outputs.any_changed == 'true' }}
        env:
          ALL_CHANGED_FILES: ${{ steps.docker_files_have_changed.outputs.all_changed_files }}
        run: |
          for file in "$ALL_CHANGED_FILES"; do
            echo "$file was changed"
          done

  build_base:
    runs-on: ubuntu-latest

    needs: check_docker_files

    if: ${{ needs.check_docker_files.outputs.docker_files_have_changed == 'true' }}

    strategy:
      matrix:
        backend:
          - name: openmp
            use_singularity: false
          - name: cuda
            use_singularity: true
          - name: hip
            use_singularity: false
          - name: sycl
            use_singularity: false

    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@v1.2.0
        with:
          tool-cache: true
          large-packages: false

      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Get Singularity
        env:
          SINGULARITY_VERSION: 3.11.2
        run: |
          wget https://github.com/sylabs/singularity/releases/download/v${{ env.SINGULARITY_VERSION }}/singularity-ce_${{ env.SINGULARITY_VERSION }}-jammy_amd64.deb
          sudo apt-get install ./singularity-ce_${{ env.SINGULARITY_VERSION }}-jammy_amd64.deb

      - name: Login in GitHub Containers Repository with Docker
        run: echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.actor }} --password-stdin

      - name: Login in GitHub Containers Repository with Singularity
        run: echo ${{ secrets.GITHUB_TOKEN }} | singularity remote login -u ${{ github.actor }} --password-stdin oras://ghcr.io

      - name: Build Docker image
        run: |
          docker build \
            -t ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }} \
            --cache-from ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }} \
            --build-arg BUILDKIT_INLINE_CACHE=1 \
            --progress=plain \
            docker/${{ matrix.backend.name }}

      - name: Push Docker image
        run: docker push ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }}

      - name: Convert Docker image to Singularity
        run: singularity build base_${{ matrix.backend.name }}.sif docker://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }}
        if: ${{ matrix.backend.use_singularity }}

      - name: Push Singularity image
        run: singularity push base_${{ matrix.backend.name }}.sif oras://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_singularity${{ needs.check_docker_files.outputs.image_name_suffix }}
        if: ${{ matrix.backend.use_singularity }}

  build:
    runs-on: ubuntu-latest

    needs:
      - check_docker_files
      - build_base

    # run this job even if build_base did not run
    if: ${{ always() && (needs.build_base.result == 'success' || needs.build_base.result == 'skipped') }}

    strategy:
      matrix:
        backend:
          - name: openmp
            c_compiler: gcc
            cxx_compiler: g++
            cmake_flags: -DKokkos_ENABLE_OPENMP=ON
          - name: cuda
            c_compiler: gcc
            cxx_compiler: g++
            cmake_flags: -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON
          - name: hip
            c_compiler: hipcc
            cxx_compiler: hipcc
            cmake_flags: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON
          - name: sycl
            c_compiler: icx
            cxx_compiler: icpx
            cmake_flags: -DKokkos_ENABLE_SYCL=ON -DKokkos_ARCH_INTEL_GEN=ON
        target:
          - name: native
            cmake_flags: ""
          - name: host_device
            cmake_flags: -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON
        exclude:
          - backend:
              name: openmp
            target:
              name: host_device

    steps:
      - name: Free Disk Space (Ubuntu)
        uses: jlumbroso/free-disk-space@v1.2.0
        with:
          tool-cache: true
          large-packages: false

      - name: Checkout built branch
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Configure
        run: |
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }} \
            cmake -B build \
            -DCMAKE_INSTALL_PREFIX=/work/install \
            -DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
            -DCMAKE_C_COMPILER=${{ matrix.backend.c_compiler }} \
            -DCMAKE_CXX_COMPILER=${{ matrix.backend.cxx_compiler }} \
            -DCMAKE_CXX_STANDARD=17 \
            -DBUILD_TESTING=ON \
            -DKokkosFFT_INTERNAL_Kokkos=ON \
            ${{ matrix.backend.cmake_flags }} \
            ${{ matrix.target.cmake_flags }}

      - name: Build
        run: |
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }} \
            cmake --build build -j $(( $(nproc) * 2 + 1 ))

      - name: Prepare artifacts
        # this is mandatory to preserve execution rights
        run: tar -cvf tests_${{ matrix.backend.name }}.tar build/
        if: ${{ matrix.target.name == 'native' }}

      - name: Save artifacts
        # use v3 as more recent versions cannot run on Ruche
        uses: actions/upload-artifact@v3
        with:
          name: tests_${{ matrix.backend.name }}
          path: tests_${{ matrix.backend.name }}.tar
        if: ${{ matrix.target.name == 'native' }}

      - name: Install
        run: |
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }} \
            cmake --install build

      - name: Configure and build test code
        run: |
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }} \
            cmake -B build_test \
            -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} \
            -DCMAKE_C_COMPILER=${{ matrix.backend.c_compiler }} \
            -DCMAKE_CXX_COMPILER=${{ matrix.backend.cxx_compiler }} \
            -DCMAKE_CXX_STANDARD=17 \
            -DCMAKE_PREFIX_PATH=/work/install \
            install_test/src
          docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }} \
            cmake --build build_test -j $(( $(nproc) * 2 + 1 ))

  test:
    runs-on: ${{ matrix.backend.runner }}

    needs:
      - check_docker_files
      - build

    # run this job even if build_base did not run
    if: ${{ always() && needs.build.result == 'success' }}

    strategy:
      matrix:
        backend:
          # run CUDA tests on Ruche supercomputer
          - name: cuda
            runner: [self-hosted, cuda]
          # run OpenMP tests on Azure server
          - name: openmp
            runner: ubuntu-latest

    steps:
      - name: Get artifacts
        # use v3 as more recent versions cannot run on Ruche
        uses: actions/download-artifact@v3
        with:
          name: tests_${{ matrix.backend.name }}

      - name: Deploy artifacts
        run: tar -xvf tests_${{ matrix.backend.name }}.tar

      - name: Run CUDA tests within Slurm job and Singularity image
        run: |
          srun --nodes=1 --time=01:00:00 -p gpua100 --gres=gpu:1 \
          singularity run --nv --bind $PWD/build:/work/build -H /work/build oras://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}_singularity${{ needs.check_docker_files.outputs.image_name_suffix }}:latest \
          ctest
        if: ${{ matrix.backend.name == 'cuda' }}

      - name: Run OpenMP tests within Docker image
        run: |
          docker run -v $PWD/build:/work/build -w /work/build ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.name }}${{ needs.check_docker_files.outputs.image_name_suffix }} \
          ctest
        if: ${{ matrix.backend.name == 'openmp' }}