.gitlab-ci-correaa.yml

# Copyright 2020-2024 Alfredo A. Correa

# Install docker (for local CI or to set up CI machine)
#   sudo apt install docker.io
# Install gitlab-runner
#    curl -L "https://packages.gitlab.com/install/repositories/runner/gitlab-runner/script.deb.sh" | sudo bash
#    sudo apt-get install gitlab-runner

image: debian:stable
# image: npneq/debian_inq_deps:bookworm

workflow:
  auto_cancel:
    on_new_commit: interruptible
  rules:
    - if: $CI_COMMIT_TAG
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'

# https://stackoverflow.com/a/78636529/225186 

variables:
  GIT_SUBMODULE_STRATEGY: recursive
  CODECOV_TOKEN: "999feb5b-a599-4d02-b9c5-46d977247f3a"
  RT_VERSION: "0.1"
  NVIDIA_DISABLE_REQUIRE: 1  # disable nvidia driver check
  SONAR_USER_HOME: "${CI_PROJECT_DIR}/.sonar"  # Defines the location of the analysis task cache
  GIT_DEPTH: "0"  # Tells git to fetch all the branches of the project, required by the analysis task

g++:  # debian-stable: default is gcc 12.2.0 as of April 2024
  stage: build
  interruptible: false
  tags:
    - non-shared
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates cmake g++ make libboost-timer-dev libboost-serialization-dev libblas-dev liblapack-dev libfftw3-dev pkg-config
    - g++ --version
    - mkdir build && cd build
    - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-D_GLIBCXX_ASSERTIONS -D_GLIBCXX_DEBUG=1"
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest -j 2 --output-on-failure -T Test

g++-m32 sanitize:
  stage: build
  image: debian:latest
  tags:
    - non-shared
    - high-bandwidth  # for boost source download
    - x86_64  # for g++-multilib in image
  interruptible: true
  script:
    - dpkg --add-architecture i386
    - apt-get -qq update
    - apt-get -qq install --no-install-recommends -y --quiet ca-certificates g++ g++-multilib cmake libboost-serialization-dev:i386 make tar wget
    - mkdir build && cd build
    - c++ --version
    - cmake .. -DCMAKE_CXX_FLAGS="-m32 -fsanitize=address -D_GLIBCXX_SANITIZE_VECTOR" -DCMAKE_BUILD_TYPE=Debug
    - cmake --build .
    - ctest -j 1 --output-on-failure
  needs: ["g++"]

arm64:
  stage: build
  image: arm64v8/debian:latest
  allow_failure: true
  tags:
    - non-shared
    - arm  # for image
  interruptible: true
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates g++ cmake make libblas-dev liblapack-dev libboost-timer-dev libboost-serialization-dev libopenmpi-dev pkg-config libfftw3-dev tar wget
    - mkdir build && cd build
    - c++ --version
    - cmake .. -DCMAKE_BUILD_TYPE=Debug
    - cmake --build .
    - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest -j 2 --output-on-failure
  needs: ["g++"]

coverage:
  stage: build
  only:
    refs:
      - master
  tags:
    - non-shared
    - docker
  allow_failure: true
  interruptible: true
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates cmake curl g++ gcovr git make lcov libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev pkg-config wget
    - mkdir build && cd build
    - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="--coverage -O0 -fno-inline -fno-inline-small-functions -fno-default-inline" -DCMAKE_EXE_LINKER_FLAGS="--coverage"
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest -j 1 --output-on-failure -T Test
    - ctest -j 1 --output-on-failure -T Coverage
    - gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml --root ${CI_PROJECT_DIR}
    # - lcov --directory . --capture --output-file coverage.info
    # - lcov --remove coverage.info '/usr/*' --output-file coverage.info
    # - lcov --list coverage.info && genhtml coverage.info
    - bash <(curl -s https://codecov.io/bash) || echo 'Codecov failed to upload'
  coverage: /^\s*Percentage\s+Coverage:\s*\d+.\d+\%/
  # coverage: /^\s*lines:\s*\d+.\d+\%/
  artifacts:
    name: ${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHA}
    expire_in: 2 days
    reports:
      coverage_report:
        coverage_format: cobertura
        path: build/coverage.xml
  needs: ["g++"]

g++ cppcheck cpplint memcheck:  # debian-stable: gcc 12 as of Dec 2023
  stage: build
  allow_failure: false
  interruptible: true
  tags:
    - non-shared
    - docker
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates cmake cppcheck cpplint g++ make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev liblapack-dev libmpich-dev pkg-config valgrind
    - g++ --version
    - cpplint --version
    - cppcheck --version
    - valgrind --version
    - mkdir build && cd build
    - export VALGRIND_EXE="valgrind --trace-children=yes --leak-check=full --track-origins=yes --show-leak-kinds=all --suppressions=.valgrind-suppressions --gen-suppressions=all --error-exitcode=1 "
    - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_CPPLINT="cpplint;--quiet" -DCMAKE_CXX_CPPCHECK="cppcheck;--enable=all;--suppress=missingIncludeSystem;--inline-suppr;--std=c++17;--check-config;--error-exitcode=1"
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - export OMPI_ALLOW_RUN_AS_ROOT=1
    - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
    - ctest -j 2 --output-on-failure -T memcheck || ctest --verbose --output-on-failure --rerun-failed -T memcheck
  needs: ["g++"]

# clang++ sanitizer:
#   stage: build
#   tags:
#     - non-shared
#     - docker
#   # image: debian:testing
#   allow_failure: true
#   interruptible: true
#   script:
#     - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates cmake clang make libasan8 libasan6 libclang-rt-dev libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev pkg-config
#     - mkdir build && cd build
#     - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=address -fno-omit-frame-pointer"  # ,undefined,pointer-compare,pointer-subtract,float-divide-by-zero -fno-omit-frame-pointer -fno-sanitize-recover=all"
#     - cmake --build . --parallel 2 || cmake --build . --verbose
#     - ctest -j 2 --output-on-failure -T Test
#   needs: ["clang++"]

# g++ sanitizer:
#   stage: build
#   # image: debian:testing
#   tags:
#     - non-shared
#     - docker
#   interruptible: true
#   script:
#     - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates cmake g++ make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev pkg-config
#     - mkdir build && cd build
#     - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined,pointer-compare,pointer-subtract,float-divide-by-zero -fno-sanitize-recover=all"
#     - cmake --build . --parallel 2 || cmake --build . --verbose
#     - ctest -j 2 --output-on-failure -T Test
#   needs: ["g++"]

g++-7:
  stage: build
  image: debian:oldoldstable  # default is gcc 8 as of Dec 2023
  tags:
    - non-shared
    - docker
  interruptible: true
  script:
    - arch
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates cmake g++-7 make libboost-serialization-dev libblas-dev libfftw3-dev libopenmpi-dev pkg-config wget
    - wget https://github.com/Kitware/CMake/releases/download/v3.27.0-rc3/cmake-3.27.0-rc3-linux-`arch`.sh --no-verbose
    - sh ./cmake-*.sh --skip-license --prefix=/usr
    - cmake --version
    - g++-7 --version
    - mkdir build && cd build
    - CXX=g++-7 cmake .. -DCMAKE_BUILD_TYPE=Debug -DDISABLE_MPI=ON
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest -j 2 --output-on-failure
  needs: ["g++"]

g++-unstable c++23 par:  # debian-testing: default is gcc 14.2.0 as of Sep 2022
  stage: build
  image: debian:unstable
  tags:
    - non-shared
    - docker
  allow_failure: true
  interruptible: true
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet g++ pkg-config cmake make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev libopenmpi-dev libtbb-dev
    - mkdir build && cd build
    - g++ --version
    - CXX=g++ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=23 -DCMAKE_CXX_FLAGS="-fimplicit-constexpr"
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure
  needs: ["g++-testing c++20"]

clang++:
  stage: build
  tags:
    - non-shared
  interruptible: false
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet clang cmake make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev liblapack-dev pkg-config
    - mkdir build && cd build
    - clang++ --version
    - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-D_GLIBCXX_ASSERTIONS -D_GLIBCXX_DEBUG=1"
    - cmake --build . --parallel 2 || cmake --build . --parallel 1 --verbose
    - ctest -j 2 --output-on-failure

clang++-16 mull-16:  # mull-17 crashes
  image: ubuntu:24.04
  stage: build
  tags:
    - x86_64  # for mull
    - non-shared
  interruptible: true
  allow_failure: false
  script:
    - apt-get -qq update
    - apt-get -qq install --no-install-recommends -y --quiet gnupg curl wget ca-certificates clang-16 cmake libclang-rt-16-dev make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev liblapack-dev pkg-config
    - curl -1sLf 'https://dl.cloudsmith.io/public/mull-project/mull-stable/setup.deb.sh' | bash
    - clang++-16 --version
    - apt-get -qq update
    - apt search mull
    - apt-get -qq install --no-install-recommends -y --quiet mull-16 libclang-cpp16
    - mkdir build && cd build
    - mull-runner-16 --version
    - CXX=clang++-16 cmake .. -DCMAKE_CXX_FLAGS="-O1 -fpass-plugin=/usr/lib/mull-ir-frontend-16 -g -grecord-command-line -fprofile-instr-generate -fcoverage-mapping"
    - cmake --build . --parallel 2 || cmake --build . --parallel 1 --verbose
    - ctest -j 2 --output-on-failure
    - cd test
    - ls *.x | xargs -n 1 sh -c 'echo $0 && ((mull-runner-16 --ld-search-path=/usr/lib/x86_64-linux-gnu $0 -test-program=ctest -- -j2 --stop-on-failure) || exit 255)'
  needs: ["clang++"]

# deb:
#   stage: build
#   script:
#     - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates cmake g++ make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev pkg-config
#     - mkdir build && cd build
#     - cmake .. -DCMAKE_BUILD_TYPE=Release
#     - cmake --build . --parallel 2 || cmake --build . --verbose
#     - ctest -j 2 --output-on-failure
#     - cpack -G DEB
#     - ls -all
#     - dpkg -i multi_all.deb
#     - ls -all /usr/include/multi
#     - c++ ../test/main.cpp
#   artifacts:
#     paths:
#       - build/multi_all.deb
#   needs: ["g++", "clang++"]

clang++-15 libc++:
  stage: build
  image: debian:stable
  tags:
    - non-shared
    - docker
  interruptible: true
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates clang-15 cmake make libboost-timer-dev libboost-serialization-dev libblas-dev libc++-dev libc++abi-dev libfftw3-dev liblapack-dev pkg-config tar wget
    - mkdir build && cd build
    - clang++-15 --version
    - CXX=clang++-15 cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_FLAGS="-stdlib=libc++ -DBOOST_NO_CXX98_FUNCTION_BASE -D_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG"
    - cmake --build .
    - ctest -j 2 --output-on-failure
  needs: ["clang++"]

clang++-latest libc++:
  stage: build
  image: debian:latest
  tags:
    - non-shared
    - docker
  interruptible: true
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates clang cmake make libboost-timer-dev libboost-serialization-dev libblas-dev libc++-dev libc++abi-dev libfftw3-dev liblapack-dev pkg-config tar wget
    - mkdir build && cd build
    - clang++ --version
    - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-stdlib=libc++ -DBOOST_NO_CXX98_FUNCTION_BASE -D_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR -D_LIBCPP_HARDENING_MODE=_LIBCPP_HARDENING_MODE_DEBUG"
    - cmake --build .
    - ctest -j 2 --output-on-failure
  needs: ["clang++"]

# - name: Install vcpkg
#   run: |
#       git clone https://github.com/microsoft/vcpkg.git
#       .\vcpkg\bootstrap-vcpkg.bat

# - name: Install Boost
#   run: .\vcpkg\vcpkg install boost-multi-array boost-timer boost-serialization

# - name: Set up Boost environment variables
#   run: |
#     echo "BOOST_ROOT=$(Get-Location)\vcpkg\installed\x64-windows" >> $env:GITHUB_ENV
#     echo "BOOST_INCLUDEDIR=$(Get-Location)\vcpkg\installed\x64-windows\include" >> $env:GITHUB_ENV
#     echo "BOOST_LIBRARYDIR=$(Get-Location)\vcpkg\installed\x64-windows\lib" >> $env:GITHUB_ENV
#     echo "PATH=$(Get-Location)\vcpkg\installed\x64-windows\bin;$env:PATH" >> $env:GITHUB_ENV

# - name: Configure CMake
#   run: cmake -S . -B build -A x64 -DCMAKE_TOOLCHAIN_FILE=.\vcpkg\scripts\buildsystems\vcpkg.cmake -DBOOST_ROOT=.\vcpkg\installed\x64-windows

vs2019-windows:
  stage: build
  only:
    refs:
      - master
  allow_failure: true
  interruptible: true
  variables:
    BOOST_ROOT: ${CI_PROJECT_DIR}/vcpkg/installed/x64-windows
    # BOOST_INCLUDEDIR: ${CI_PROJECT_DIR}/vcpkg/installed/x64-windows/include
    # BOOST_LIBRARYDIR: ${CI_PROJECT_DIR}/vcpkg/installed/x64-windows/lib
  script:
    - choco --version
    - choco install -y visualstudio2019community poshgit
    - git clone --depth=1 https://github.com/microsoft/vcpkg.git
    - .\vcpkg\bootstrap-vcpkg.bat
    - .\vcpkg\vcpkg install boost-multi-array boost-timer
    - mkdir build
    - cmake --version
    - cmake -S . -B build -DCMAKE_CXX_STANDARD=23
    - cmake --build build --config Release --parallel 2 --verbose
    - $env:Path += ";${CI_PROJECT_DIR}/vcpkg/installed/x64-windows/bin"
    - ctest --test-dir build --output-on-failure -C Release
  tags:
    - saas-windows-medium-amd64
  needs: ["clang++", "g++"]
  timeout: 30 minutes

# vs2022-windows:
#   stage: build
#   allow_failure: true
#   script:
#     # - choco --version
#     # - choco upgrade -y chocolatey
#     - choco --version
#     - choco install -y visualstudio2022community
#     - choco install -y boost-msvc-14.2 --version 1.74.0  # --params "/StaticBuild"
#     - mkdir build
#     - cmake --version
#     - cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -A x64  # -DBOOST_ROOT="C:\local\boost_1_74_0" -DBOOST_LIBRARYDIR="C:\local\boost_1_74_0\lib64-msvc-14.2"  # c:/local/boost_1_74_0/lib64-msvc-14.2
#     - cmake --build build --config Release
#     - $env:Path += ';C:\local\boost_1_74_0\lib64-msvc-14.2'  # C:\local\boost_1_74_0;  ;C:\Program Files\CMake\bin
#     - ctest --test-dir build --output-on-failure -C Release
#   tags:
#     - saas-windows-medium-amd64
#   timeout: 30  # minutes
#   # rules:
#   #   - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
#   needs: ["vs2019-windows"]

# vs2017-windows:
#   stage: build
#   allow_failure: true
#   script:
#     - choco --version
#     - choco install -y visualstudio2017community
#     - choco install -y boost-msvc-14.2 --version 1.74.0  # --params "/StaticBuild"
#     - mkdir build
#     - cmake --version
#     - cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -A x64  # -DBOOST_ROOT="C:\local\boost_1_74_0" -DBOOST_LIBRARYDIR="C:\local\boost_1_74_0\lib64-msvc-14.2"  # c:/local/boost_1_74_0/lib64-msvc-14.2
#     - cmake --build build --config Release
#     - $env:Path += ';C:\local\boost_1_74_0\lib64-msvc-14.2'  # C:\local\boost_1_74_0;  ;C:\Program Files\CMake\bin
#     - ctest --test-dir build --output-on-failure -C Release
#   tags:
#     - saas-windows-medium-amd64
#   # rules:
#   #   - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
#   needs: ["vs2019-windows"]

# clang++-macos:
#   interruptible: true
#   stage: build
# # image: macos-14-xcode-15
# # variables:
# #   HOMEBREW_NO_AUTO_UPDATE: 1
#   tags:
#     - shared-macos-amd64
#     # - saas-macos-medium-m1
#     # - saas-macos-large-m2pro 
#   allow_failure: true
#   script:
#     #- apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates clang cmake make libblas-dev libc++-dev libc++abi-dev libfftw3-dev pkg-config tar wget
#     #- wget https://downloads.sourceforge.net/project/boost/boost/1.72.0/boost_1_72_0.tar.gz --no-verbose # 1.70 doesn't install boost timer well, there is no version 1.72 in sourceforge
#     #- tar -xf boost_1_72_0.tar.gz 
#     #- cd boost_1_72_0
#     #- ./bootstrap.sh --with-toolset=clang
#     #- ./b2 toolset=clang cxxflags=-stdlib=libc++ linkflags=-stdlib=libc++ --with-serialization --with-test --with-timer install -j4 # libc++ only works with boost test compiled with libc++
#     #- cd ..
#     - mkdir build && cd build
#     - c++ --version
#     - CXX=c++ cmake .. -DCMAKE_BUILD_TYPE=Debug  # -DCMAKE_CXX_FLAGS="-stdlib=libc++ -DBOOST_NO_CXX98_FUNCTION_BASE -D_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR -D_LIBCPP_ENABLE_DEBUG_MODE=1 -D_LIBCPP_DEBUG_RANDOMIZE_UNSPECIFIED_STABILITY=1 -D_LIBCPP_DEBUG_STRICT_WEAK_ORDERING_CHECK=1"
#     - cmake --build .
#     - ctest -j 2 --output-on-failure
#   needs: ["clang++"]

clang++-m32:
  stage: build
  image: debian:latest
  tags:
    - non-shared
    - high-bandwidth  # for boost source download
    - x86_64  # for g++-multilib in image
  interruptible: true
  script:
    - dpkg --add-architecture i386
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates clang g++ g++-multilib git cmake make libboost-serialization-dev:i386 tar wget
    - mkdir build && cd build
    - c++ --version
    - CXX=clang++ cmake .. -DCMAKE_CXX_FLAGS="-m32" -DCMAKE_BUILD_TYPE=Debug
    - cmake --build .
    - ctest -j 2 --output-on-failure
  needs: ["clang++"]

clang++-latest tidy iwyu:
  stage: build
  image: debian:latest  # clang 18 as of Apr 2024
  tags:
    - non-shared
    - docker
  interruptible: true
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet clang clang-tidy cmake iwyu make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev liblapack-dev pkg-config
    - mkdir build && cd build
    - clang++ --version
    - clang-tidy --version
    - iwyu --version
    - CXX=clang++ cmake .. -DCMAKE_CXX_STANDARD=20 -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_CLANG_TIDY=clang-tidy -DCMAKE_CXX_INCLUDE_WHAT_YOU_USE="iwyu;-Xiwyu;--mapping_file=$PWD/../.iwyu-test.imp;-Xiwyu;--no_fwd_decls;-Xiwyu;--error"
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest -j 2 --output-on-failure
  needs: ["clang++"]

g++-testing c++20:
  stage: build
  image: debian:testing
  tags:
    - non-shared
    - docker
  interruptible: true
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet g++ pkg-config cmake make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev
    - mkdir build && cd build
    - cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_FLAGS=-fimplicit-constexpr
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest --output-on-failure
  needs: ["g++"]

clang++-19-unstable libc++ c++26 boost_1_87:
  stage: build
  image: debian:unstable  # clang 17 as of March 2024
  tags:
    - non-shared
    - docker
    - high-bandwidth
  interruptible: true
  script:  # clang 17 doesn't work with gcc 13 libstd
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates clang clang-19 cmake make libblas-dev libc++-19-dev libc++abi-19-dev libfftw3-dev pkg-config tar wget
    - wget https://downloads.sourceforge.net/project/boost/boost/1.87.0/boost_1_87_0.tar.gz --no-verbose
    - tar -xf boost_1_87_0.tar.gz 
    - cd boost_1_87_0
    - ./bootstrap.sh --with-toolset=clang
    - ./b2 toolset=clang cxxflags=-stdlib=libc++ linkflags=-stdlib=libc++ --with-serialization --with-test --with-timer install -j4  # libc++ only works with boost test compiled with libc++
    - cd ..
    - mkdir build && cd build
    - CXX=clang++-19 cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_STANDARD=26 -DCMAKE_CXX_FLAGS="-stdlib=libc++ -D_LIBCPP_ENABLE_DEBUG_MODE=1 -D_LIBCPP_DEBUG_RANDOMIZE_UNSPECIFIED_STABILITY=1 -D_LIBCPP_DEBUG_STRICT_WEAK_ORDERING_CHECK=1"  # -DBoost_NO_SYSTEM_PATHS=ON
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest --output-on-failure
  needs: ["clang++-latest libc++", "g++-testing c++20"]

clang++-oldoldstable:
  stage: build
  image: debian:oldoldstable  # clang 7.0.1 as of Jan 2025
  tags:
    - non-shared
    - docker
  interruptible: true
  script:
    - arch
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates clang cmake make libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev pkg-config wget
    - wget --no-verbose -O cmake-install.sh https://github.com/Kitware/CMake/releases/download/v3.30.0-rc4/cmake-3.30.0-rc4-linux-`arch`.sh  # https://cmake.org/files/v3.18/cmake-3.18.0-rc1-Linux-`arch`.sh --no-verbose
    - sh cmake-install.sh --skip-license --prefix=/usr
    - mkdir build && cd build 
    - clang++ --version
    - CXX=clang++ cmake .. -DCMAKE_BUILD_TYPE=Release
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest -j 2 --output-on-failure
  needs: ["clang++"]

oneapi:
  stage: build
  image: intel/oneapi-hpckit:2023.0.0-devel-ubuntu22.04  # Intel(R) oneAPI DPC++/C++ Compiler 2023.0.0 (2023.0.0.20221201)
  tags:
    - non-shared
    - large-disk-space
    - x86_64
  interruptible: true
  script:
    - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates cmake curl libboost-serialization-dev libboost-timer-dev libblas-dev liblapack-dev libfftw3-dev make pkg-config
    - mkdir build && cd build
    - icpx --version
    - CXX=icpx cmake .. -DCMAKE_BUILD_TYPE=Release -DDISABLE_MPI=ON  # Intel MPI doesn't work on CI
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest --output-on-failure

oneapi-2022.2:
  stage: build
  image: intel/oneapi-hpckit:2022.2-devel-ubuntu20.04
  tags:
    - non-shared
    - large-disk-space
    - x86_64
  allow_failure: true
  interruptible: true
  script:
    - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates cmake curl g++ libboost-serialization-dev libboost-timer-dev libblas-dev liblapack-dev libfftw3-dev make pkg-config
    - mkdir build && cd build
    - icpx --version
    - CXX=icpx CXXFLAGS="-DPSTL_USE_PARALLEL_POLICIES=0" cmake .. -DCMAKE_BUILD_TYPE=Release -DDISABLE_MPI=ON  # Intel MPI doesn't work on CI
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest --parallel 2 || ctest --verbose --rerun-failed --output-on-failure
  needs: ["oneapi"]

oneapi-latest c++20:
  stage: build
  image: intel/oneapi-hpckit:latest  # icpx --version (2023.2.0.20230721) as of Dec 2023
  allow_failure: true
  tags:
    - non-shared
    - large-disk-space
    - high-bandwidth
    - x86_64
  interruptible: true
  script:
    # - wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
    # - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list
    - apt-get update && apt-get install --no-install-recommends -y --quiet ca-certificates cmake curl g++ libboost-serialization-dev libboost-timer-dev libblas-dev liblapack-dev libfftw3-dev make pkg-config
    - mkdir build && cd build
    - icpx --version
    - CXX=icpx cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 -DDISABLE_MPI=ON  # Intel MPI doesn't work on CI
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest --parallel 2 || ctest --verbose --rerun-failed --output-on-failure
  needs: ["oneapi"]

nvhpc:
  stage: build
  image: nvcr.io/nvidia/nvhpc:22.11-devel-cuda11.8-ubuntu22.04  # https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nvhpc/tags
  tags:
    - non-shared
    - large-disk-space
    - x86_64
  script:
    - apt-get update && apt-get install --no-install-recommends -y cmake make libboost-serialization-dev
    - /opt/nvidia/hpc_sdk/Linux_x86_64/2022/compilers/bin/nvc++ --version
    - mkdir build && cd build
    - CXX=/opt/nvidia/hpc_sdk/Linux_x86_64/2022/compilers/bin/nvc++ cmake ..
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - export OMPI_ALLOW_RUN_AS_ROOT=1
    - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
    - ctest --parallel 2 --output-on-failure || ctest --verbose --rerun-failed --output-on-failure

nvhpc-22.7:
  stage: build
  image: nvcr.io/nvidia/nvhpc:22.7-devel-cuda11.7-ubuntu22.04  # https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nvhpc/tags
  tags:
    - non-shared
    - large-disk-space
    - x86_64
  interruptible: true
  script:
    - apt-get update && apt-get install --no-install-recommends -y cmake make libboost-timer-dev libboost-serialization-dev
    - /opt/nvidia/hpc_sdk/Linux_x86_64/2022/compilers/bin/nvc++ --version
    - mkdir build && cd build
    - CXX=/opt/nvidia/hpc_sdk/Linux_x86_64/2022/compilers/bin/nvc++ cmake ..  # TODO(correaa) add -DCMAKE_BUILD_TYPE=Release
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure
  needs: ["nvhpc"]

nvhpc-24.11 c++20 par:
  stage: build
  image: nvcr.io/nvidia/nvhpc:24.11-devel-cuda12.6-ubuntu24.04  # nvcr.io/nvidia/nvhpc:24.7-devel-cuda12.5-ubuntu24.04  # https://catalog.ngc.nvidia.com/orgs/nvidia/containers/nvhpc/tags
  tags:
    - non-shared
    - large-disk-space
    - x86_64
  interruptible: true
  script:
    - apt-get update && apt-get install --no-install-recommends -y cmake make libboost-timer-dev libboost-serialization-dev libfftw3-dev pkg-config
    - /opt/nvidia/hpc_sdk/Linux_x86_64/2024/compilers/bin/nvc++ --version
    - mkdir build && cd build
    - CXX=/opt/nvidia/hpc_sdk/Linux_x86_64/2024/compilers/bin/nvc++ cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_FLAGS="-stdpar=multicore"
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest --output-on-failure
  needs: ["nvhpc"]

cuda:
  stage: build
  allow_failure: false
  image: nvcr.io/nvidia/cuda:12.0.1-devel-ubuntu22.04
  tags:
    - non-shared
    - nvidia-gpu
  interruptible: true
  script:
    - nvidia-smi
    - apt-get -qq update && apt-get install --no-install-recommends -y cmake g++-12 wget pkg-config make libboost-serialization-dev libboost-timer-dev libblas-dev libfftw3-dev
    - mkdir build && cd build
    - g++-12 --version
    - /usr/local/cuda/bin/nvcc --version
    - CUDACXX=/usr/local/cuda/bin/nvcc cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES=61 -DCMAKE_CUDA_HOST_COMPILER=g++-12 -DCMAKE_CUDA_FLAGS="-allow-unsupported-compiler"
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest || ctest --rerun-failed --output-on-failure
  needs: ["g++"]

cuda-11.4.3:
  stage: build
  image: nvcr.io/nvidia/cuda:11.4.3-devel-ubuntu20.04
  tags:
    - non-shared
    - nvidia-gpu
    - x86_64
  interruptible: true
  script:
    - nvidia-smi
    # - export CUDA_VISIBLE_DEVICES=2
    - apt-get -qq update
    - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y cmake wget pkg-config make libboost-serialization-dev libboost-timer-dev libblas-dev libfftw3-dev libtbb-dev
    - wget --no-verbose -O cmake-install.sh https://github.com/Kitware/CMake/releases/download/v3.27.0-rc3/cmake-3.27.0-rc3-linux-x86_64.sh
    - sh ./cmake-install.sh --skip-license --prefix=/usr
    - cmake --version
    - mkdir build && cd build
    - /usr/local/cuda/bin/nvcc --version
    - CUDACXX=/usr/local/cuda/bin/nvcc cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_CUDA=1 -DCMAKE_CUDA_ARCHITECTURES=61
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest -j 2 --output-on-failure
  needs: ["cuda"]

culang++-16 cuda-11.4.3:
  stage: build
  image: nvcr.io/nvidia/cuda:11.4.3-devel-ubuntu20.04
  tags:
    - non-shared
    - nvidia-gpu
  interruptible: true
  script:
    - nvidia-smi
    - apt-get -qq update
    - DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y cmake wget pkg-config make libboost-serialization-dev libboost-timer-dev libblas-dev libfftw3-dev
    - apt-get install --no-install-recommends -y lsb-release software-properties-common
    - wget https://github.com/Kitware/CMake/releases/download/v3.27.0-rc3/cmake-3.27.0-rc3-linux-x86_64.sh --no-verbose
    - sh ./cmake-3.27.0-rc3-linux-x86_64.sh --skip-license --prefix=/usr
    - cmake --version
    - wget https://apt.llvm.org/llvm.sh
    - chmod u+x llvm.sh
    - ./llvm.sh 16
    - mkdir build && cd build
    - clang++-16 --version
    - cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_CUDA=1 -DCMAKE_CUDA_COMPILER=clang++-16 -DCMAKE_CXX_COMPILER=clang++-16
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest -j 2 --output-on-failure
  needs: ["cuda", "clang++"]

cuda-12.6.3:
  stage: build
  allow_failure: true
  image: nvcr.io/nvidia/cuda:12.6.3-devel-ubuntu24.04  # nvcr.io/nvidia/cuda:12.5.0-devel-ubuntu22.04
  tags:
    - non-shared
    - nvidia-gpu
  interruptible: true
  script:
    - nvidia-smi
    - apt-get -qq update
    - apt-get install --no-install-recommends -y cmake g++ wget pkg-config make libboost-serialization-dev libboost-timer-dev libblas-dev libfftw3-dev
    - cmake --version
    - mkdir build && cd build
    - g++ --version
    - /usr/local/cuda/bin/nvcc --version
    - CUDACXX=/usr/local/cuda/bin/nvcc cmake .. -DCMAKE_BUILD_TYPE=Release -DENABLE_CUDA=1 -DCMAKE_CXX_STANDARD=20 -DCMAKE_CUDA_STANDARD=20 -DCMAKE_CUDA_ARCHITECTURES=61 -DCMAKE_CUDA_HOST_COMPILER=g++
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest || ctest --rerun-failed --output-on-failure
  needs: ["cuda"]

culang++-19 cuda-12.6.3 tidy:  # https://catalog.ngc.nvidia.com/orgs/nvidia/containers/cuda/tags
  stage: build
  image: nvcr.io/nvidia/cuda:12.6.3-devel-ubuntu24.04
  allow_failure: false
  tags:
    - non-shared
    - nvidia-gpu
    - high-bandwidth
  interruptible: true
  script:
    - nvidia-smi
    - apt-get -qq update && apt-get install --no-install-recommends -y cmake wget pkg-config make libboost-serialization-dev libboost-timer-dev libblas-dev libfftw3-dev
    - apt-get install --no-install-recommends -y lsb-release software-properties-common
    - wget https://apt.llvm.org/llvm.sh
    - chmod u+x llvm.sh
    - ./llvm.sh 19
    - apt-get install --no-install-recommends -y clang-19 clang-tidy-19
    - mkdir build && cd build
    - clang++-19 --version
    - clang-tidy-19 --version
    - cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_FLAGS="-Wno-unknown-cuda-version" -DCMAKE_CXX_CLANG_TIDY=clang-tidy-19 -DENABLE_CUDA=1 -DCMAKE_CUDA_COMPILER=clang++-19 -DCMAKE_CXX_COMPILER=clang++-19 -DCMAKE_CUDA_ARCHITECTURES=61
    - cmake --build . --parallel 2 || cmake --build . --verbose
    - ctest -j 2 --output-on-failure
  needs: ["cuda", "clang++"]

rocm:
  stage: build
  image: rocm/dev-ubuntu-24.04  # rocm/dev-ubuntu-22.04
  allow_failure: true
  tags:
    - non-shared
    - docker
    - high-bandwidth  # for download rocm extras
    - x86_64  # for rocm image
  interruptible: true
  script:
    - apt-get -qq update
    - apt-get install --no-install-recommends -y cmake wget pkg-config make libboost-serialization-dev libboost-timer-dev libblas-dev libfftw3-dev wget gpg
    - apt-get install --no-install-recommends -y rocthrust-dev hipblas-dev hipfft-dev rocm-device-libs
    - /opt/rocm/bin/hipconfig --full
    - HIPCC_VERBOSE=1 /opt/rocm/bin/hipcc --version
    - cmake --version
    - wget https://github.com/Kitware/CMake/releases/download/v3.27.0-rc3/cmake-3.27.0-rc3-linux-x86_64.sh --no-verbose
    - sh ./cmake-3.27.0-rc3-linux-x86_64.sh --skip-license --prefix=/usr
    - cmake --version
    - mkdir build && cd build
    - export PATH=/opt/rocm/bin:/opt/rocm/llvm/bin:${PATH}
    - export CMAKE_PREFIX_PATH=/opt/rocm:${CMAKE_PREFIX_PATH}
    - export ROCM_PATH=/opt/rocm
    - export HIP_PATH=/opt/rocm
    - cmake .. -DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ -DCMAKE_CXX_FLAGS="-DMULTI_USE_HIP" -DCMAKE_HIP_FLAGS="-DMULTI_USE_HIP" -DENABLE_HIP=1 -DCMAKE_HIP_ARCHITECTURES=gfx90a  # TODO(correaa) add CMAKE_BUILD_TYPE
    - cmake --build . --parallel 2 || cmake --build . --verbose --parallel 1
    - ctest || ctest --rerun-failed --output-on-failure || echo "ctest failed, probably due to lack of hardware"
  needs: ["clang++", "g++"]

circle:
  stage: build
  allow_failure: true
  tags:
    - non-shared
    - x86_64  # for circle executable
  script:
    - apt-get -qq update
    - apt-get -qq install --no-install-recommends -y --quiet ca-certificates clang cmake g++-11 gzip libboost-timer-dev libboost-serialization-dev libblas-dev libelf1 libfftw3-dev make pkg-config tar wget
    - mkdir -p build_latest ; cd build_latest
    - wget https://www.circle-lang.org/linux/build_202.tgz --no-verbose
    - tar -zxvf build_*.tgz
    - cd ..
    - ./build_latest/circle --version
    - mkdir build && cd build
    - CXX=`pwd`/../build_latest/circle cmake .. -DCMAKE_BUILD_TYPE=Debug -DENABLE_CIRCLE=1
    - cmake --build . --parallel 4 || make VERBOSE=1
    - ctest -j 2 --output-on-failure

circle-latest c++20:
  stage: build
  allow_failure: true
  tags:
    - non-shared
    - x86_64  # for circle executable
  interruptible: true
  script:
    - apt-get -qq update
    - apt-get -qq install --no-install-recommends -y --quiet ca-certificates clang cmake g++ gzip libboost-timer-dev libboost-serialization-dev libblas-dev libelf1 libfftw3-dev make pkg-config tar wget
    - mkdir -p build_latest ; cd build_latest
    - wget https://www.circle-lang.org/linux/build_latest.tgz --no-verbose
    - tar -zxvf build_*.tgz
    - cd ..
    - ls
    - ./build_latest/circle --version
    - mkdir build && cd build
    - CXX=`pwd`/../build_latest/circle cmake .. -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_EXTENSIONS=OFF -DCMAKE_CXX_STANDARD=20 -DENABLE_CIRCLE=1
    - cmake --build . --parallel 2 || make VERBOSE=1
    - ctest -j 2 --output-on-failure
  needs: ["circle"]

inq:
  stage: test
  # image: debian:stable
  tags:
    - non-shared
    - large-memory-space
  interruptible: true
  script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y ca-certificates cmake g++ git gfortran libopenmpi-dev libblas-dev libboost-filesystem-dev libboost-iostreams-dev libboost-serialization-dev libfftw3-dev libhdf5-dev liblapack-dev make ninja-build pkg-config python3-dev 
    - export PREFIX=`mktemp -d`
    - git clone --recurse-submodules https://gitlab.com/npneq/inq.git  # --branch update-multi-get
    - cd inq
    - git submodule update
    - cd external_libs/multi
    - git checkout $CI_COMMIT_SHA  # check that multi repo is mirrored correctly
    - cd ../..
    - mkdir build && cd build
    - cmake .. -G Ninja --install-prefix=$PREFIX -DCMAKE_BUILD_TYPE=Release
    - cmake --build . || cmake --build . --parallel 1
    - apt-get -qq --no-install-recommends -y install python3-numpy
    - cmake --install .
    - export OMPI_ALLOW_RUN_AS_ROOT=1
    - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
    - export OMPI_MCA_btl_vader_single_copy_mechanism=none
    - export OMPI_MCA_rmaps_base_oversubscribe=1
    - ctest -j 2 --output-on-failure --timeout 600
    - INQ_EXEC_ENV="mpirun --oversubscribe -n 4" ctest --output-on-failure --timeout 600
  needs: ["g++"]

inq cuda:
  allow_failure: false
  stage: test
  image: nvcr.io/nvidia/cuda:12.0.1-devel-ubuntu22.04  # nvcr.io/nvidia/cuda:11.8.0-devel-ubuntu22.04
  tags:
    - non-shared
    - nvidia-gpu
  interruptible: true
  before_script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y ca-certificates cmake git gfortran libopenmpi-dev libblas-dev libboost-filesystem-dev libboost-iostreams-dev libboost-serialization-dev libfftw3-dev libhdf5-dev liblapack-dev pkg-config python3-dev
  script:
    - nvidia-smi
    # - export CUDA_VISIBLE_DEVICES=0,1
    - __nvcc_device_query
    - export PREFIX=`mktemp -d`
    - git clone --recurse-submodules https://gitlab.com/npneq/inq.git  # --branch update-multi-get
    - cd inq
    - cd external_libs/multi
    - git checkout $CI_COMMIT_SHA
    - cd ../..
    - rm -f cmake/FindNCCL.cmake  # disable NCCL workaround
    - mkdir build && cd build
    - /usr/local/cuda/bin/nvcc --version
    - CUDACXX=/usr/local/cuda/bin/nvcc cmake .. --install-prefix=$PREFIX -DENABLE_CUDA=1 -DCMAKE_BUILD_TYPE=Release -DCMAKE_CUDA_ARCHITECTURES=61  # =75  # =80
    - cmake --build . --parallel 4 || cmake --build . --parallel 1
    - apt-get -qq --no-install-recommends -y install python3-numpy
    - cmake --install .
    - export OMPI_ALLOW_RUN_AS_ROOT=1
    - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
    - export OMPI_MCA_btl_vader_single_copy_mechanism=none
    - export OMPI_MCA_rmaps_base_oversubscribe=1
    - ctest -j 2 --output-on-failure --timeout 2400
    - INQ_EXEC_ENV="mpirun --oversubscribe -n 4" ctest --output-on-failure --timeout 2400
  timeout: 2 hours 30 minutes
  needs: ["cuda", "inq"]

inq rocm:
  stage: test
  # image: rocm/dev-ubuntu-22.04
  image: rocm/dev-ubuntu-24.04
  allow_failure: true
  tags:
    - non-shared
    - large-disk-space
    - x86_64  # for image
  interruptible: true
  script:
    - apt-get -qq update
    - apt-get -qq install --no-install-recommends -y ca-certificates cmake git gfortran gpg hipblas-dev hipfft-dev libopenmpi-dev libblas-dev libboost-filesystem-dev libboost-iostreams-dev libboost-serialization-dev libfftw3-dev libhdf5-dev liblapack-dev make pkg-config python3-dev rocthrust-dev rocm-device-libs wget
    - /opt/rocm/bin/hipconfig --full
    - HIPCC_VERBOSE=1 /opt/rocm/bin/hipcc --version
    - cmake --version
    - wget https://github.com/Kitware/CMake/releases/download/v3.27.0-rc3/cmake-3.27.0-rc3-linux-x86_64.sh --no-verbose
    - sh ./cmake-3.27.0-rc3-linux-x86_64.sh --skip-license --prefix=/usr
    - cmake --version
    - export PREFIX=`mktemp -d`
    - git clone --recurse-submodules https://gitlab.com/npneq/inq.git  # --branch update-multi-get
    - cd inq
    - cd external_libs/multi
    - git checkout $CI_COMMIT_SHA
    - cd ../..
    - mkdir build && cd build
    - export PATH=/opt/rocm/bin:/opt/rocm/llvm/bin:${PATH}
    - export CMAKE_PREFIX_PATH=/opt/rocm:${CMAKE_PREFIX_PATH}
    - export ROCM_PATH=/opt/rocm
    - export HIP_PATH=/opt/rocm
    - cmake .. -DCMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ --install-prefix=$PREFIX -DENABLE_HIP=1 -DCMAKE_HIP_ARCHITECTURES=gfx90a
    - cmake --build . --parallel 4 || cmake --build . --parallel 1
    - cmake --install .
    - export OMPI_ALLOW_RUN_AS_ROOT=1
    - export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
    - export OMPI_MCA_btl_vader_single_copy_mechanism=none
    - export OMPI_MCA_rmaps_base_oversubscribe=1
    - INQ_EXEC_ENV="mpirun --oversubscribe -n 4" ctest --output-on-failure --timeout 2400 || echo "ctest failed, probably due to lack of hardware"
  timeout: 2 hours 30 minutes
  needs: ["rocm", "inq"]

qmcpack:
  stage: test
  image: debian:latest
  tags:
    - non-shared
    - docker
  interruptible: true
  before_script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y ca-certificates cmake g++ git gfortran libblas-dev libboost-serialization-dev libfftw3-dev libhdf5-dev liblapack-dev libopenmpi-dev make numdiff pkg-config python3 python3-h5py python3-numpy python3-mpi4py python3-scipy libxml2-dev
  script:
    - git clone --depth=1 https://github.com/QMCPACK/qmcpack.git  # https://github.com/correaa/qmcpack.git --branch correaa-patch-use-adl-get
    - cd qmcpack
    - git config --global user.email "alfredo.correa@gmail.com" && git config --global user.name "Alfredo Correa"
    - git rm -r external_codes/boost_multi/multi && git commit -m "remove multi subtree"
    - git subtree add --squash -P external_codes/boost_multi/multi $CI_REPOSITORY_URL $CI_COMMIT_SHA || git subtree add --squash -P external_codes/boost_multi/multi $CI_REPOSITORY_URL # e.g. https://gitlab.com/correaa/boost-multi.git
    - cd build
    - cmake -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DBUILD_AFQMC=1 -DQMC_MIXED_PRECISION=1 -DCMAKE_BUILD_TYPE=Debug -DMPIEXEC_PREFLAGS="--allow-run-as-root;--bind-to;none" ..
    - make ppconvert afqmc test_afqmc_matrix test_afqmc_numerics test_afqmc_slaterdeterminantoperations test_afqmc_walkers test_afqmc_hamiltonians test_afqmc_hamiltonian_operations test_afqmc_phmsd test_afqmc_wfn_factory test_afqmc_prop_factory test_afqmc_estimators qmc-afqmc-performance
    - ctest -R ppconvert --output-on-failure
    - ctest -R afqmc     --output-on-failure
  needs: ["g++"]

qmcpack cuda-12.3.1:
  stage: test
  image: nvcr.io/nvidia/cuda:12.3.1-devel-ubuntu22.04
  tags:
    - non-shared
    - nvidia-gpu
    - high-bandwidth
  interruptible: true
  before_script:
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y ca-certificates cmake git libopenmpi-dev cmake g++ git gfortran libblas-dev libboost-serialization-dev libfftw3-dev libhdf5-dev liblapack-dev libopenmpi-dev make numdiff pkg-config python3 python3-h5py python3-numpy python3-mpi4py python3-scipy libxml2-dev
  script:
    - nvidia-smi
    - git clone --depth 1 https://github.com/QMCPACK/qmcpack.git  # https://github.com/correaa/qmcpack.git --branch correaa-patch-use-adl-get
    - cd qmcpack
    - git config --global user.email "alfredo.correa@gmail.com" && git config --global user.name "Alfredo Correa"
    - git rm -r external_codes/boost_multi/multi && git commit -m "remove multi subtree"
    - git subtree add --squash -P external_codes/boost_multi/multi $CI_REPOSITORY_URL $CI_COMMIT_SHA # e.g. https://gitlab.com/correaa/boost-multi.git
    - cd build
    - nvcc --version
    - __nvcc_device_query
    - CUDACXX=/usr/local/cuda/bin/nvcc cmake .. -DCMAKE_C_COMPILER=mpicc -DCMAKE_CXX_COMPILER=mpicxx -DBUILD_AFQMC=1 -DQMC_CXX_STANDARD=17 -DQMC_GPU=cuda -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc -DCMAKE_CUDA_HOST_COMPILER=g++ -DCMAKE_CXX_FLAGS="-Wno-deprecated -Wno-deprecated-declarations" -DCMAKE_CUDA_ARCHITECTURES=75  # =80
    - make -j 4 ppconvert afqmc test_afqmc_matrix test_afqmc_numerics test_afqmc_slaterdeterminantoperations test_afqmc_walkers test_afqmc_hamiltonians test_afqmc_hamiltonian_operations test_afqmc_phmsd test_afqmc_wfn_factory test_afqmc_prop_factory test_afqmc_estimators qmc-afqmc-performance
    - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest -R ppconvert --output-on-failure
    - OMPI_ALLOW_RUN_AS_ROOT=1 OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 ctest -R afqmc     --output-on-failure
  needs: ["qmcpack","cuda"]

# sonar cloud
# from instructions
# another example here: https://github.com/sonarsource-cfamily-examples/linux-cmake-gitlab-ci-sc/blob/main/.gitlab-ci.yml

sonar:
  stage: build
  allow_failure: true
  only:
    refs:
      - master
  tags:
    - non-shared
    - docker
    - x86_64  # for executable
  interruptible: true
  script:
    # Run the build inside the build wrapper
    - apt-get -qq update && apt-get -qq install --no-install-recommends -y --quiet ca-certificates cmake curl unzip g++ gcovr make lcov libboost-timer-dev libboost-serialization-dev libblas-dev libfftw3-dev pkg-config wget
    - g++ --version
    - mkdir build
    # Download sonar-scanner, from https://docs.sonarsource.com/sonarqube-cloud/advanced-setup/ci-based-analysis/sonarscanner-cli/
    - wget https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-6.2.1.4610-linux-x64.zip -O sonar-scanner.zip
    - unzip -o sonar-scanner.zip
    - mv sonar-scanner-6.2.1.4610-linux-x64 sonar-scanner
    # - Download build-wrapper
    - curl -sSLo ./build-wrapper-linux-x86.zip "${SONAR_HOST_URL}/static/cpp/build-wrapper-linux-x86.zip"
    - unzip -oj build-wrapper-linux-x86.zip -d ./build-wrapper
    - cmake -S . -B build -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_CXX_FLAGS="--coverage -O0 -fno-inline -fno-inline-small-functions -fno-default-inline" -DCMAKE_EXE_LINKER_FLAGS="--coverage"
    - build-wrapper/build-wrapper-linux-x86-64 --out-dir bw-output cmake --build build/ --verbose
    - cd build
    - ctest -j 1 --output-on-failure -T Test
    - ctest -j 1 --output-on-failure -T Coverage
    - gcovr --xml-pretty --exclude-unreachable-branches --print-summary -o coverage.xml --root ${CI_PROJECT_DIR}
    - cd ..
    - sonar-scanner/bin/sonar-scanner -Dsonar.host.url="${SONAR_HOST_URL}" -Dsonar.token="${SONAR_TOKEN}" -Dsonar.python.version=3.12.3 -Dsonar.cfamily.compile-commands=build/compile_commands.json `#-Dsonar.cfamily.build-wrapper-output=bw-output` -Dsonar.cfamily.gcov.reportsPath="build/"
  needs: ["g++"]