Merge pull request #137 from boeschf/oomph

Oomph
ghex-org · Dec 20, 2021 · 6591c29 · 6591c29
2 parents 9a7a401 + 9fb21e5
commit 6591c29
Show file tree

Hide file tree

Showing 274 changed files with 13,676 additions and 26,144 deletions.
diff --git a/.clang-format b/.clang-format
@@ -0,0 +1,93 @@
+---
+Language:        Cpp
+# BasedOnStyle:  LLVM
+#AccessModifierOffset: -4
+AccessModifierOffset: -2
+AlignAfterOpenBracket: DontAlign
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: true
+AlignConsecutiveMacros: true
+#AlignConsecutiveAssignments: false
+#AlignConsecutiveDeclarations: false
+#AlignConsecutiveMacros: false
+AlignEscapedNewlines: Right
+AlignOperands: true
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: false
+AllowAllParametersOfDeclarationOnNextLine: false
+BinPackArguments: true
+BinPackParameters: true
+BreakBeforeBraces: Allman
+#BreakBeforeBraces: Attach
+#BreakBeforeBraces: Custom
+#BraceWrapping:
+#    AfterCaseLabel: true
+#    AfterClass: true
+# BreakBeforeBinaryOperators: All
+# ConstructorInitializerAllOnOneLineOrOnePerLine: false
+BreakConstructorInitializers: BeforeComma
+ConstructorInitializerIndentWidth: 0
+BreakInheritanceList: BeforeComma
+#AllowShortBlocksOnASingleLine: Always
+AllowShortBlocksOnASingleLine: true
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Always
+AllowShortLambdasOnASingleLine: All
+AllowShortLoopsOnASingleLine: true
+#AlwaysBreakAfterReturnType: None
+AlwaysBreakAfterReturnType: TopLevelDefinitions
+# PenaltyReturnTypeOnItsOwnLine: 1
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: Yes
+BreakBeforeTernaryOperators: true
+BreakInheritanceList: BeforeComma
+BreakStringLiterals: false
+ColumnLimit: 100
+#ColumnLimit: 120
+# CommentPragmas
+CompactNamespaces: false
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+#--DeriveLineEnding: true
+#--DerivePointerAlignment: true
+FixNamespaceComments: true
+# ForEachMacros: [for_each]
+# IncludeBlocks: Regroup
+IndentCaseLabels: true
+#IndentGotoLabels: false
+#IndentPPDirectives: BeforeHash
+IndentPPDirectives: None
+IndentWidth: 4
+IndentWrappedFunctionNames: false
+KeepEmptyLinesAtTheStartOfBlocks: false
+MaxEmptyLinesToKeep: 1
+NamespaceIndentation: None
+PointerAlignment: Left
+ReflowComments: false
+SortIncludes: false
+SortUsingDeclarations: false
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: false
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+#SpaceBeforeSquareBrackets: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+# SpacesInConditionalStatement: false
+SpacesInContainerLiterals: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+#Standard: Latest
+TabWidth: 4
+UseTab: Never
+
+
+
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -7,10 +7,43 @@ on:
       - master
 
 jobs:
-  build-cpu:
+  build-cpu-mpi:
     runs-on: ubuntu-latest
     container: ghcr.io/gridtools/gridtools-base:gcc-9-ucx-mpi-atlas-parmetis
+    steps:
+      - uses: actions/checkout@v1
+      - name: Configure
+        run: |
+          echo "WORKFLOW"
+          export SHLVL=1
+          env
+          ulimit -a
+          ipcs -lm
+          df
+          cat /proc/cpuinfo
+          mkdir -p build && cd build
+          eckit_DIR=/opt/eckit Atlas_DIR=/opt/atlas cmake .. \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DGHEX_WITH_TESTING=ON \
+            -DGHEX_BUILD_BENCHMARKS=ON \
+            -DGHEX_BUILD_FORTRAN=ON \
+            -DGHEX_ENABLE_ATLAS_BINDINGS=ON \
+            -DGHEX_ENABLE_PARMETIS_BINDINGS=ON \
+            -DMETIS_INCLUDE_DIR=/opt/metis/include \
+            -DMETIS_LIB_DIR=/opt/metis/lib \
+            -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include \
+            -DPARMETIS_LIB_DIR=/opt/parmetis/lib \
+            -DGHEX_TRANSPORT_BACKEND=MPI \
+            -DGHEX_USE_GPU=OFF \
+            -DMPIEXEC_PREFLAGS=--oversubscribe
+      - name: Build
+        run: cmake --build build --parallel 2
+      - name: Execute tests
+        run: cd build && export SHLVL=1 && export OMPI_ALLOW_RUN_AS_ROOT=1 && export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 && export CTEST_OUTPUT_ON_FAILURE=1 && env && ctest
 
+  build-cpu-ucx:
+    runs-on: ubuntu-latest
+    container: ghcr.io/gridtools/gridtools-base:gcc-9-ucx-mpi-atlas-parmetis
     steps:
       - uses: actions/checkout@v1
       - name: Configure
@@ -23,16 +56,63 @@ jobs:
           df
           cat /proc/cpuinfo
           mkdir -p build && cd build
-          eckit_DIR=/opt/eckit Atlas_DIR=/opt/atlas cmake .. -DUCX_DIR=/opt/ucx -DMETIS_INCLUDE_DIR=/opt/metis/include -DMETIS_LIB_DIR=/opt/metis/lib -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include -DPARMETIS_LIB_DIR=/opt/parmetis/lib -DCMAKE_BUILD_TYPE=Debug -DGHEX_USE_UCP=ON -DGHEX_ENABLE_ATLAS_BINDINGS=ON -DGHEX_ENABLE_PARMETIS_BINDINGS=ON -DGHEX_BUILD_FORTRAN=ON -DCMAKE_Fortran_FLAGS="-cpp -ffree-line-length-none -fcoarray=single" -DGHEX_BUILD_TESTS=ON -DGHEX_BUILD_BENCHMARKS=ON -DMPIEXEC_PREFLAGS=-oversubscribe
+          eckit_DIR=/opt/eckit Atlas_DIR=/opt/atlas cmake .. \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DGHEX_WITH_TESTING=ON \
+            -DGHEX_BUILD_BENCHMARKS=ON \
+            -DGHEX_BUILD_FORTRAN=ON \
+            -DGHEX_ENABLE_ATLAS_BINDINGS=ON \
+            -DGHEX_ENABLE_PARMETIS_BINDINGS=ON \
+            -DMETIS_INCLUDE_DIR=/opt/metis/include \
+            -DMETIS_LIB_DIR=/opt/metis/lib \
+            -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include \
+            -DPARMETIS_LIB_DIR=/opt/parmetis/lib \
+            -DGHEX_TRANSPORT_BACKEND=UCX \
+            -DGHEX_USE_GPU=OFF \
+            -DMPIEXEC_PREFLAGS=--oversubscribe \
+            -DUCX_DIR=/opt/ucx 
       - name: Build
         run: cmake --build build --parallel 2
       - name: Execute tests
         run: cd build && export SHLVL=1 && export OMPI_ALLOW_RUN_AS_ROOT=1 && export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 && export CTEST_OUTPUT_ON_FAILURE=1 && env && ctest
 
-  build-gpu-amd:
+  build-cpu-mpi-clang:
     runs-on: ubuntu-latest
     container: ghcr.io/gridtools/gridtools-base:gcc-9-ucx-mpi-atlas-parmetis-hip
+    steps:
+      - uses: actions/checkout@v1
+      - name: Configure
+        run: |
+          echo "WORKFLOW"
+          export SHLVL=1
+          env
+          ulimit -a
+          ipcs -lm
+          df
+          cat /proc/cpuinfo
+          mkdir -p build && cd build
+          CXX=/opt/rocm/llvm/bin/clang++ eckit_DIR=/opt/eckit Atlas_DIR=/opt/atlas cmake .. \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DGHEX_WITH_TESTING=ON \
+            -DGHEX_BUILD_BENCHMARKS=ON \
+            -DGHEX_BUILD_FORTRAN=OFF \
+            -DGHEX_ENABLE_ATLAS_BINDINGS=ON \
+            -DGHEX_ENABLE_PARMETIS_BINDINGS=ON \
+            -DMETIS_INCLUDE_DIR=/opt/metis/include \
+            -DMETIS_LIB_DIR=/opt/metis/lib \
+            -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include \
+            -DPARMETIS_LIB_DIR=/opt/parmetis/lib \
+            -DGHEX_TRANSPORT_BACKEND=MPI \
+            -DGHEX_USE_GPU=OFF \
+            -DMPIEXEC_PREFLAGS=--oversubscribe
+      - name: Build
+        run: cmake --build build --parallel 2
+      - name: Execute tests
+        run: cd build && export SHLVL=1 && export OMPI_ALLOW_RUN_AS_ROOT=1 && export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 && export CTEST_OUTPUT_ON_FAILURE=1 && env && ctest
 
+  build-cpu-ucx_clang:
+    runs-on: ubuntu-latest
+    container: ghcr.io/gridtools/gridtools-base:gcc-9-ucx-mpi-atlas-parmetis-hip
     steps:
       - uses: actions/checkout@v1
       - name: Configure
@@ -45,14 +125,63 @@ jobs:
           df
           cat /proc/cpuinfo
           mkdir -p build && cd build
-          CXX=/opt/rocm/llvm/bin/clang++ cmake .. -DUSE_GPU=ON -DUCX_DIR=/opt/ucx -DMETIS_INCLUDE_DIR=/opt/metis/include -DMETIS_LIB_DIR=/opt/metis/lib -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include -DPARMETIS_LIB_DIR=/opt/parmetis/lib -DCMAKE_BUILD_TYPE=Debug -DGHEX_USE_UCP=ON -DGHEX_ENABLE_PARMETIS_BINDINGS=ON -DGHEX_BUILD_FORTRAN=ON -DCMAKE_Fortran_FLAGS="-cpp -ffree-line-length-none -fcoarray=single" -DGHEX_BUILD_TESTS=ON -DGHEX_BUILD_BENCHMARKS=ON
+          CXX=/opt/rocm/llvm/bin/clang++ eckit_DIR=/opt/eckit Atlas_DIR=/opt/atlas cmake .. \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DGHEX_WITH_TESTING=ON \
+            -DGHEX_BUILD_BENCHMARKS=ON \
+            -DGHEX_BUILD_FORTRAN=OFF \
+            -DGHEX_ENABLE_ATLAS_BINDINGS=ON \
+            -DGHEX_ENABLE_PARMETIS_BINDINGS=ON \
+            -DMETIS_INCLUDE_DIR=/opt/metis/include \
+            -DMETIS_LIB_DIR=/opt/metis/lib \
+            -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include \
+            -DPARMETIS_LIB_DIR=/opt/parmetis/lib \
+            -DGHEX_TRANSPORT_BACKEND=UCX \
+            -DGHEX_USE_GPU=OFF \
+            -DMPIEXEC_PREFLAGS=--oversubscribe \
+            -DUCX_DIR=/opt/ucx 
+      - name: Build
+        run: cmake --build build --parallel 2
+      - name: Execute tests
+        run: cd build && export SHLVL=1 && export OMPI_ALLOW_RUN_AS_ROOT=1 && export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1 && export CTEST_OUTPUT_ON_FAILURE=1 && env && ctest
+
+  build-gpu-amd:
+    runs-on: ubuntu-latest
+    container: ghcr.io/gridtools/gridtools-base:gcc-9-ucx-mpi-atlas-parmetis-hip
+    steps:
+      - uses: actions/checkout@v1
+      - name: Configure
+        run: |
+          echo "WORKFLOW"
+          export SHLVL=1
+          env
+          ulimit -a
+          ipcs -lm
+          df
+          cat /proc/cpuinfo
+          mkdir -p build && cd build
+          CXX=/opt/rocm/llvm/bin/clang++ cmake .. \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DGHEX_WITH_TESTING=ON \
+            -DGHEX_BUILD_BENCHMARKS=ON \
+            -DGHEX_BUILD_FORTRAN=OFF \
+            -DGHEX_ENABLE_ATLAS_BINDINGS=OFF \
+            -DGHEX_ENABLE_PARMETIS_BINDINGS=ON \
+            -DMETIS_INCLUDE_DIR=/opt/metis/include \
+            -DMETIS_LIB_DIR=/opt/metis/lib \
+            -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include \
+            -DPARMETIS_LIB_DIR=/opt/parmetis/lib \
+            -DGHEX_TRANSPORT_BACKEND=UCX \
+            -DGHEX_USE_GPU=ON \
+            -DGHEX_GPU_TYPE=AMD \
+            -DMPIEXEC_PREFLAGS=--oversubscribe \
+            -DUCX_DIR=/opt/ucx 
       - name: Build
         run: cmake --build build --parallel 2
 
   build-gpu-nvidia:
     runs-on: ubuntu-latest
     container: ghcr.io/gridtools/gridtools-base:gcc-9-ucx-mpi-atlas-parmetis-cuda-11
-
     steps:
       - uses: actions/checkout@v1
       - name: Configure
@@ -65,6 +194,22 @@ jobs:
           df
           cat /proc/cpuinfo
           mkdir -p build && cd build
-          cmake .. -DUSE_GPU=ON -DUCX_DIR=/opt/ucx -DMETIS_INCLUDE_DIR=/opt/metis/include -DMETIS_LIB_DIR=/opt/metis/lib -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include -DPARMETIS_LIB_DIR=/opt/parmetis/lib -DCMAKE_BUILD_TYPE=Debug -DGHEX_USE_UCP=ON -DGHEX_ENABLE_PARMETIS_BINDINGS=ON -DGHEX_BUILD_FORTRAN=ON -DCMAKE_Fortran_FLAGS="-cpp -ffree-line-length-none -fcoarray=single" -DGHEX_BUILD_TESTS=ON -DGHEX_BUILD_BENCHMARKS=ON
+          cmake .. \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DGHEX_WITH_TESTING=ON \
+            -DGHEX_BUILD_BENCHMARKS=ON \
+            -DGHEX_BUILD_FORTRAN=OFF \
+            -DGHEX_ENABLE_ATLAS_BINDINGS=OFF \
+            -DGHEX_ENABLE_PARMETIS_BINDINGS=ON \
+            -DMETIS_INCLUDE_DIR=/opt/metis/include \
+            -DMETIS_LIB_DIR=/opt/metis/lib \
+            -DPARMETIS_INCLUDE_DIR=/opt/parmetis/include \
+            -DPARMETIS_LIB_DIR=/opt/parmetis/lib \
+            -DGHEX_TRANSPORT_BACKEND=UCX \
+            -DGHEX_USE_GPU=ON \
+            -DGHEX_GPU_TYPE=NVIDIA \
+            -DMPIEXEC_PREFLAGS=--oversubscribe \
+            -DUCX_DIR=/opt/ucx 
       - name: Build
         run: cmake --build build --parallel 2
+