github-ci.yml

name: Github-CI

on:
  # Dispatch this workflow whenever master or dev get a PR or commit
  push:
    branches : ["dev","master"]
  pull_request:
    branches: ["dev","master"]
  # ... or when the workflow is started manually
  workflow_dispatch:

concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true

jobs:
  build_libraries:
    # Build libraries for the current version of the docker image
    # (to be used in any subsequent compilation and run jobs on said image)
    runs-on: ubuntu-latest
    container: ursg/vlasiator_ci:20241101_1

    steps:
      - name: Checkout source
        uses: actions/checkout@v4
      - name: Run library build script
        run: ./build_libraries.sh
      - name: Build libraries tar
        run: tar --zstd -cvf libraries.tar.zstd libraries/
      - name: Upload libraries as artifact
        uses: actions/upload-artifact@v4
        with:
          name: libraries
          path: libraries.tar.zstd
          retention-days: 5
          if-no-files-found: error

  build_libraries_appleM1:
    # Build libraries on macos with M1 hardware, to test both macOS and aarch64 compatibility
    runs-on: macos-14

    steps:
      - name: Checkout source
        uses: actions/checkout@v4
      - name: Setup MPI
        uses: mpi4py/setup-mpi@v1
        with:
          mpi: 'openmpi'
      - name: Setup openmp-capable llvm
        run: |
          brew install llvm libomp
          echo "CXX=/opt/homebrew/opt/llvm/bin/clang++" >> "$GITHUB_ENV"
          echo "OMPI_CXX=/opt/homebrew/opt/llvm/bin/clang++" >> "$GITHUB_ENV"
          echo "MPI_CXX=mpic++" >> "$GITHUB_ENV"
          echo "CC=/opt/homebrew/opt/llvm/bin/clang" >> "$GITHUB_ENV"
          echo "OMPI_CC=/opt/homebrew/opt/llvm/bin/clang" >> "$GITHUB_ENV"
          echo "MPI_CC=mpicc" >> "$GITHUB_ENV"
      - name: Run library build script
        run: |
          . ./build_libraries.sh appleM1
      - name: Build libraries tar
        run: tar --zstd -cvf libraries-appleM1.tar.zstd libraries-appleM1
      - name: Upload libraries as artifact
        uses: actions/upload-artifact@v4
        with:
          name: libraries-appleM1
          path: libraries-appleM1.tar.zstd
          retention-days: 5
          if-no-files-found: error

  build_libraries_riscv:
    # Build libraries for the current version of the docker image
    # (to be used in any subsequent compilation and run jobs on said image)
    runs-on: risc-v

    steps:
      - name: Checkout source
        uses: actions/checkout@v4
        with:
          submodules: true
      - name: Clean workspace
        run: |
          rm -rf libraries library-build libraries-arriesgado.tar.gz
      - name: Submit library job
        run: |
          #srun --interactive -p arriesgado-jammy -J vlasiator-libs -n 1 -c 4 -t 01:00:00 bash -lc 'module load openmpi; ./build_libraries.sh arriesgado'
          ssh synth-hca 'source /etc/profile.d/lmod.sh; export PATH=$PATH:$HOME/bin; module load llvm/cross/EPI-0.7-development; cd '$GITHUB_WORKSPACE'; ./build_libraries.sh arriesgado'
      - name: Build libraries tar
        run: tar -czvf libraries-arriesgado.tar.gz libraries-arriesgado/
      - name: Upload libraries as artifact
        uses: actions/upload-artifact@v4
        with:
          name: libraries-arriesgado
          path: libraries-arriesgado.tar.gz
          retention-days: 5
          if-no-files-found: error

  build_production:
    # Build Vlasiator with production flags
    runs-on: ubuntu-latest
    container: ursg/vlasiator_ci:20241101_1
    needs: build_libraries

    steps:
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: true
    - name: Download libraries
      uses: actions/download-artifact@v4
      with:
        name: libraries
    - name: Unpack libraries
      run: tar --zstd -xvf libraries.tar.zstd
    - uses: ursg/gcc-problem-matcher@master
    - name: Compile vlasiator (Release build)
      run: |
        VLASIATOR_ARCH=github_actions make clean
        VLASIATOR_ARCH=github_actions make -j 3
    - name: Upload release binary
      uses: actions/upload-artifact@v4
      with:
        name: vlasiator-release
        path: vlasiator
        if-no-files-found: error


  build_testpackage:
    # Build Vlasiator with testpackage flags, on the carrington cluster
    # (for subsequent running of the integration test package)
    runs-on: carrington

    steps:
    - name: Clean workspace
      run: |
        RUN_STRING=$( cat << MORO
        rm -rf libraries library-build testpackage
        rm -f libraries.tar.zst testpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt
        rm -f *.xml
        MORO
        )
        srun -M carrington bash -c "$RUN_STRING"
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: true
    - name: Make clean
      run: VLASIATOR_ARCH=carrington_gcc_openmpi make clean
    - uses: ursg/gcc-problem-matcher@master
    - name: Compile vlasiator (Testpackage build)
      run: |
        export VLASIATOR_ARCH=carrington_gcc_openmpi
        srun -M carrington --job-name CI_tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/13.2.0; module load OpenMPI/4.1.6-GCC-13.2.0 ; module load PMIx/4.2.6-GCCcore-13.2.0; module load PAPI/7.1.0-GCCcore-13.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 testpackage; sleep 10s'
    - name: Make sure the output binary is visible in lustre
      uses: nick-fields/retry@v3
      with:
        timeout_seconds: 15
        max_attempts: 3
        retry_on: error
        command: ls vlasiator
    - name: Upload testpackage binary
      uses: actions/upload-artifact@v4
      with:
        name: vlasiator-testpackage
        path: vlasiator
        if-no-files-found: error
    #- name: Upload build log
    #  uses: actions/upload-artifact@v4
    #  with:
    #    name: Testpackage build log
    #    path: build.log

  build_riscv:
    runs-on: risc-v
    needs: build_libraries_riscv

    steps:
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: true
    - name: Clean workspace
      run: |
        rm -rf libraries library-build libraries-arriesgado.tar.gz
        rm -f stdout.txt stderr.txt metrics.txt
        rm -f *.xml
    - name: Download libraries
      uses: actions/download-artifact@v4
      with:
        name: libraries-arriesgado
    - name: Unpack libraries
      run: tar -xzvf libraries-arriesgado.tar.gz
    - name: Make clean
      run: VLASIATOR_ARCH=arriesgado make clean
    - name: Compile vlasiator (RISC-V)
      run: |
        #srun --interactive -p arriesgado-jammy -J vlasiator_build -n 1 -c 4 --pty -t 01:00:00 bash -lc 'module load boost papi openmpi; export VLASIATOR_ARCH=arriesgado; make -j4'
        ssh synth-hca 'source /etc/profile.d/lmod.sh; export PATH=$PATH:$HOME/bin; module load llvm/cross/EPI-0.7-development; export VLASIATOR_ARCH=arriesgado; cd '$GITHUB_WORKSPACE'; make -j 12'
    - name: Upload riscv binary
      uses: actions/upload-artifact@v4
      with:
        name: vlasiator-riscv
        path: vlasiator
        if-no-files-found: error

  build_appleM1:
    runs-on: macos-14
    needs: build_libraries_appleM1

    steps:
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: true
    - name: Setup MPI
      uses: mpi4py/setup-mpi@v1
      with:
        mpi: 'openmpi'
    - name: Setup openmp-capable llvm
      run: |
        brew install llvm libomp
        echo "CXX=/opt/homebrew/opt/llvm/bin/clang++" >> "$GITHUB_ENV"
        echo "OMPI_CXX=/opt/homebrew/opt/llvm/bin/clang++" >> "$GITHUB_ENV"
        echo "CC=/opt/homebrew/opt/llvm/bin/clang" >> "$GITHUB_ENV"
        echo "OMPI_CC=/opt/homebrew/opt/llvm/bin/clang" >> "$GITHUB_ENV"
    - name: Install boost
      run: |
        brew install boost
        ln -s /opt/homebrew/Cellar/boost/* /opt/homebrew/Cellar/boost/latest
    - name: Download libraries
      uses: actions/download-artifact@v4
      with:
        name: libraries-appleM1
    - name: Unpack libraries
      run: tar --zstd -xvf libraries-appleM1.tar.zstd
    - name: Make clean
      run: VLASIATOR_ARCH=appleM1 make clean
    - name: Compile vlasiator (Release build)
      run: |
        VLASIATOR_ARCH=appleM1 make clean
        VLASIATOR_ARCH=appleM1 make -j 3

  build_tools:
    # Build vlsvdiff and vlsvextract for testpackage use
    runs-on: carrington

    steps:
    - name: Clean workspace
      run: |
        RUN_STRING=$( cat << MORO
        rm -rf libraries library-build testpackage
        rm -f libraries.tar.zstd testpackage_check_description.txt testpackage-output.tar.gz metrics.txt stdout.txt stderr.txt testpackage_output_variables.txt
        rm -f *.xml
        MORO
        )
        srun -M carrington bash -c "$RUN_STRING"
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ursg/gcc-problem-matcher@master
    - name: Make clean
      run: VLASIATOR_ARCH=carrington_gcc_openmpi make clean
    - name: Compile tools
      run: |
        export VLASIATOR_ARCH=carrington_gcc_openmpi
        srun -M carrington --job-name CI_tp_compile --interactive --nodes=1 -n 1 -c 16 --mem=40G -p short -t 0:10:0 bash -c 'module purge; module load GCC/11.2.0; module load OpenMPI/4.1.1-GCC-11.2.0 ; module load PMIx/4.1.0-GCCcore-11.2.0; module load PAPI/6.0.0.1-GCCcore-11.2.0; export VLASIATOR_ARCH=carrington_gcc_openmpi; make -j 16 vlsvextract vlsvdiff'
    - name: Upload tools binaries
      uses: actions/upload-artifact@v4
      with:
        name: vlasiator-tools
        path: |
          vlsvextract_DP
          vlsvdiff_DP
        if-no-files-found: error

  run_testpackage:
    # Run the testpackage on the carrington cluster
    runs-on: carrington
    needs: [build_testpackage, build_tools]
    continue-on-error: true

    steps:
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: false
    - name: Download testpackage binary
      uses: actions/download-artifact@v4
      with:
        name: vlasiator-testpackage
    - name: Download tools
      uses: actions/download-artifact@v4
      with:
        name: vlasiator-tools
    - name: Run testpackage
      id: run
      run: |
        chmod +x $GITHUB_WORKSPACE/vlasiator
        chmod +x $GITHUB_WORKSPACE/vlsv*_DP
        cd testpackage
        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib
        sbatch -W -o testpackage_run_output.txt ./small_test_carrington_github_ci.sh
        PARSE_OUTPUT_CMD=$( cat << MORO
        echo "Job finished, checking output."
        cat testpackage_run_output.txt
        cat $GITHUB_STEP_SUMMARY > $GITHUB_WORKSPACE/testpackage_check_description.txt
        cd $GITHUB_WORKSPACE
        ls -halB testpackage_check_description.txt
        tar -czf testpackage-output.tar.gz testpackage_check_description.txt testpackage_output_variables.txt
        MORO
        )
        srun --job-name CI_package_results -M carrington -N 1 -c 1 --mem=1G bash -c "$PARSE_OUTPUT_CMD"
        if [ -f $GITHUB_WORKSPACE/testpackage_failed ]; then
          # Fail this step if any test failed.
          exit 1
        fi
    - name: Scancel dangling job upon cancellation
      if: cancelled()
      run: |
        # Try accessing the job id echoed by the job script.
        scancel ${{ steps.run.outputs.SLURM_JOB_ID }}
    - name: Make sure the output tarball is visible in lustre
      uses: nick-fields/retry@v3
      with:
        timeout_seconds: 15
        max_attempts: 3
        retry_on: error
        command: ls $GITHUB_WORKSPACE/testpackage-output.tar.gz
    - name: Upload testpackage output
      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: testpackage-output
        path: testpackage-output.tar.gz
      # Note: Testpackage output is further processed in the pr_report.yml workflow
      # (to produce Checks against pull requests)

  build_ionosphereTests:
    # Build IonosphereSolverTests miniApp
    runs-on: ubuntu-latest
    container: ursg/vlasiator_ci:20241101_1
    needs: build_libraries
    steps:
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: true
          #    - name: Download libraries
          #      uses: actions/download-artifact@v4
          #      with:
          #        name: libraries
          #    - name: Unpack libraries
          #      run: tar --zstd -xvf libraries.tar.zstd
          #    - uses: ursg/gcc-problem-matcher@master
    - name: Download libraries
      uses: actions/download-artifact@v4
      with:
        name: libraries
    - name: Unpack libraries
      run: tar --zstd -xvf libraries.tar.zstd
    - name: Compile ionosphereSolverTests
      run: |
        cd mini-apps/ionosphereSolverTests/
        VLASIATOR_ARCH=github_actions make -j 3
    - name: Upload ionosphereTest binaries
      uses: actions/upload-artifact@v4
      with:
        name: ionosphereTests
        path: |
          mini-apps/ionosphereSolverTests/main
          mini-apps/ionosphereSolverTests/differentialFlux
          mini-apps/ionosphereSolverTests/sigmaProfiles
        if-no-files-found: error

  run_ionosphere_LFMtest:
    runs-on: ubuntu-latest
    container: ursg/vlasiator_ci:20241101_1
    needs: build_ionosphereTests
    steps:
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: false
    - name: Download ionosphereTests
      uses: actions/download-artifact@v4
      with:
        name: ionosphereTests
    - name: checkout Analysator
      run: |
        apt update
        apt -y install python3 python3-numpy python3-matplotlib gnuplot-nox
        cd mini-apps/ionosphereSolverTests/
        git clone https://github.com/fmihpc/analysator
    - name: Run LFMtest
      run: |
        chmod +x $GITHUB_WORKSPACE/main
        cd mini-apps/ionosphereSolverTests/
        OMP_NUM_THREADS=1 $GITHUB_WORKSPACE/main -N 2000 -r 40 90 -r 50 90 -r 60 80 -sigma 10 -fac merkin2010 -gaugeFix equator45 -o LFMtest.vlsv -maxIter 10000
        export PYTHONPATH=$PYTHONPATH:analysator
        export PTNONINTERACTIVE=1
        python3 ./lfmformat.py LFMtest.vlsv > lfmtest.dat
        ./plot_LFMtest.gp
    - name: Run atmospheric profile test
      run: |
        chmod +x $GITHUB_WORKSPACE/sigmaProfiles
        cd mini-apps/ionosphereSolverTests/
        $GITHUB_WORKSPACE/sigmaProfiles 1e6 1.16046e7 > atmosphere.dat
        $GITHUB_WORKSPACE/sigmaProfiles 1e6 1.16046e8 > atmosphere10.dat
        $GITHUB_WORKSPACE/sigmaProfiles 1e6 1.16046e9 > atmosphere100.dat
        ./plotAtmosphere.gp
    - name: Upload ionosphereTest result images
      uses: actions/upload-artifact@v4
      with:
        name: ionosphereTestResults
        path: |
          mini-apps/ionosphereSolverTests/LFMtest.png
          mini-apps/ionosphereSolverTests/atmosphere.png
        if-no-files-found: error

  check_cfg_files:
    runs-on: ubuntu-latest
    container: ursg/vlasiator_ci:20241101_1
    needs: [build_libraries, build_production]
    steps:
    - name: Checkout source
      uses: actions/checkout@v4
      with:
        submodules: false
    - name: Download libraries
      uses: actions/download-artifact@v4
      with:
        name: libraries
    - name: Unpack libraries
      run: tar --zstd -xvf libraries.tar.zstd
    - name: Download release binary
      uses: actions/download-artifact@v4
      with:
        name: vlasiator-release
    - name: Make release binary executable
      run: |
        chmod +x $GITHUB_WORKSPACE/vlasiator
    - name: Check cfg files in projects/
      run: |
        retval=0
        set +e
        for cfg in `find projects -name \*.cfg`; do
          echo -n "Testing cfg file $cfg... "
          PRINT_ONLY_ERRORS=true LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib tools/check_vlasiator_cfg.sh $GITHUB_WORKSPACE/vlasiator $cfg 2>&1 > cfgtest.out
          if [ $? != 0 ]; then
            if `echo $cfg | grep -sq unsupported`; then
              echo "failed (unsurprising)"
            else
              echo "FAILED:";
              cat cfgtest.out;
              retval=127
            fi
          else
            echo "ok."
          fi;
        done
        exit $retval
    - name: Check cfg files in samples/
      run: |
        retval=0
        set +e
        for cfg in `find samples -name \*.cfg`; do
          echo -n "Testing cfg file $cfg... "
          PRINT_ONLY_ERRORS=true LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib tools/check_vlasiator_cfg.sh $GITHUB_WORKSPACE/vlasiator $cfg 2>&1 > cfgtest.out
          if [ $? != 0 ]; then
            echo "FAILED:";
            cat cfgtest.out;
            retval=127
          else
            echo "ok."
          fi;
        done
        exit $retval
    - name: Check cfg files in testpackage/
      run: |
        retval=0
        set +e
        for cfg in `find testpackage -name \*.cfg`; do
          echo -n "Testing cfg file $cfg... "
          PRINT_ONLY_ERRORS=true LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/libraries/lib tools/check_vlasiator_cfg.sh $GITHUB_WORKSPACE/vlasiator $cfg 2>&1 > cfgtest.out
          if [ $? != 0 ]; then
            echo "FAILED:";
            cat cfgtest.out;
            retval=127
          else
            echo "ok."
          fi;
        done
        exit $retval