Version 1.0.0

.clang-format

@@ -0,0 +1,62 @@
+Language:        Cpp
+# BasedOnStyle:  Google
+AccessModifierOffset: -1
+AlignAfterOpenBracket: true
+AlignEscapedNewlinesLeft: true
+AlignOperands:   true
+AlignTrailingComments: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AlwaysBreakAfterDefinitionReturnType: false
+AlwaysBreakTemplateDeclarations: true
+AlwaysBreakBeforeMultilineStrings: true
+BreakBeforeBinaryOperators: None
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BinPackParameters: true
+BinPackArguments: true
+ColumnLimit:     120
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth: 4
+DerivePointerAlignment: false
+ExperimentalAutoDetectBinPacking: false
+IndentCaseLabels: true
+IndentWrappedFunctionNames: false
+IndentFunctionDeclarationAfterType: false
+MaxEmptyLinesToKeep: 1
+KeepEmptyLinesAtTheStartOfBlocks: false
+NamespaceIndentation: None
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakBeforeFirstCallParameter: 1
+PenaltyBreakComment: 300
+PenaltyBreakString: 1000
+PenaltyBreakFirstLessLess: 120
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Right
+SpacesBeforeTrailingComments: 2
+Cpp11BracedListStyle: true
+Standard:        Cpp11
+IndentWidth:     2
+TabWidth:        2
+UseTab:          Never
+BreakBeforeBraces: Attach
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpacesInAngles:  false
+SpaceInEmptyParentheses: false
+SpacesInCStyleCastParentheses: false
+SpaceAfterCStyleCast: false
+SpacesInContainerLiterals: true
+SpaceBeforeAssignmentOperators: true
+ContinuationIndentWidth: 4
+CommentPragmas:  '^ IWYU pragma:'
+ForEachMacros:   [ foreach, Q_FOREACH, BOOST_FOREACH ]
+SpaceBeforeParens: ControlStatements
+DisableFormat:   false

CONTRIBUTING.md

@@ -0,0 +1,15 @@
+# Contributing to MPI-CorrBench
+
+MPI-Corrbench is a community effort and we appreciate contributions.
+
+## Contributing
+
+If you want to contribute, please fork the project and open a pull request.
+
+## Issues
+
+Please use the issue tracker for problems, errors, or feature requests and suggestions for improvements.
+
+## Further Questions
+
+If you have further questions, please don't hesitate to reach out to Tim Jammer or JP Lehr.

CONTRIBUTORS

@@ -0,0 +1,13 @@
+# This is the official list of MPI correctness benchmark authors for copyright purposes.
+
+# Names should be added to this file as one of
+#     Organization's name
+#     Individual's name <Affiliation> <*optional:* submission email address>
+
+# Please keep the list sorted according to affiliation and first contribution.
+
+TU Darmstadt, Darmstadt, Germany
+Usman Ahmad <Scientific Computing>
+Jan-Patrick Lehr <Scientific Computing>
+Tim Jammer <Scientific Computing>
+Moritz Fischer <Scientific Computing>

LICENSE.txt

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2020, Scientific Computing, TU Darmstadt
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

README.md

@@ -0,0 +1,25 @@
+# MPI-CorrBench · [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) 
+
+MPI-CorrBench \[[CORRBE21](#ref-Corrbe21)\] enables a structured comparison of the different available tools for MPI correctness checking  w.r.t. various types of errors.
+
+## Usage
+Provide the Following environment Variables:
+* `MPI_CORRECTNESS_BM_DIR` full path to the local directory of this repo
+* `MPI_CORRECTNESS_BM_EXPERIMENT_DIR` full path to the directory where the tools should be executed
+
+### Running the benchmarks
+* Adapt the SLURM configuration in `scripts/SLURM_header.in`
+* for each tool a script file `scripts/<TOOL>/execute_tool.sh` exists. You may need to adopt it to use the tool on your system
+* run `scripts/run_measurement.sh 1` to generate the jobscript and submit them via sbatch (the argument given is the number of repetition for each test case)
+* after all jobs are finished use `scripts/gather_results.sh` to gather all the results, which will _override_ previous data in `output` (such as our results)
+
+### Visualize the results
+The visualizations presented in \[[CORRBE21](#ref-Corrbe21)\] can be generated by the usage of the provided scripts. The scripts don't take any arguments and read the data in `output` to generate the visualization.
+
+#### References
+<table style="border:0px">
+<tr>
+    <td valign="top"><a name="ref-Corrbe21"></a>[CORRBE21]</td>
+    <td>Lehr, Jan-Patrick and Jammer, Tim and Bischof, Christian:
+      MPI-CorrBench: Towards an MPI Correctness Benchmark Suite 2021.</td>
+</tr>

collect-error-cases.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python
+
+# populates the error-cases.json
+# by collecting error specifications from the *.c files in ./micro-benches
+
+import os
+import json
+
+ARG_ERROR = 'ArgError-'
+ARG_MISMATCH = 'ArgMismatch-'
+MISPLACED_CALL = 'MisplacedCall-'
+MISSING_CALL = 'MissingCall-'
+
+def main():
+    dir_name = "./micro-benches"
+    levels = ["0-level", "1-level", "2-level"]
+    categories = ["coll", "pt2pt", "rma", "usertypes"]
+    all_files = []
+    notes_str= '< Currently none >'
+    total_errs = 0
+    total_corrs = 0
+    the_total = 0
+    total_tcs = {}
+    for c in categories:
+      total_tcs[c] = {}
+
+    # Print overview of the number of test cases for the different categories
+    base_dir = dir_name + '/' + levels[0]
+    for cat in categories:
+      tcs = {ARG_ERROR: 0, ARG_MISMATCH: 0, MISPLACED_CALL: 0, MISSING_CALL: 0}
+      file_names = os.listdir(base_dir + '/' + cat)
+      file_names.sort()
+
+      for f in file_names:
+        all_files.append(base_dir + '/' + cat + '/' + f)
+        found = False
+        if ARG_ERROR in f:
+          tcs[ARG_ERROR] = tcs[ARG_ERROR] + 1
+          found = True
+        if ARG_MISMATCH in f:
+          tcs[ARG_MISMATCH] = tcs[ARG_MISMATCH] + 1
+          found = True
+        if MISPLACED_CALL in f:
+          tcs[MISPLACED_CALL] = tcs[MISPLACED_CALL] + 1
+          found = True
+        if MISSING_CALL in f:
+          tcs[MISSING_CALL] = tcs[MISSING_CALL] + 1
+          found = True
+
+        if not found:
+          print('Did not find: ' + base_dir + '/' + cat + '/' + f)
+
+      total_tcs[cat][ARG_ERROR] = tcs[ARG_ERROR]
+      total_tcs[cat][ARG_MISMATCH] = tcs[ARG_MISMATCH]
+      total_tcs[cat][MISPLACED_CALL] = tcs[MISPLACED_CALL]
+      total_tcs[cat][MISSING_CALL] = tcs[MISSING_CALL]
+      print('=== Category: {cat} ===\n{the_dict}\n >> All cases should statically determinable.'.format(cat=cat,the_dict=tcs))
+
+    # Code Duplication // FIXME
+    for cat in categories:
+      tcs = {ARG_ERROR: 0, ARG_MISMATCH: 0, MISPLACED_CALL: 0, MISSING_CALL: 0}
+      file_names = os.listdir(base_dir + '/conflo/' + cat)
+      file_names.sort()
+
+      for f in file_names:
+        found = False
+        if ARG_ERROR in f:
+          tcs[ARG_ERROR] = tcs[ARG_ERROR] + 1
+          total_tcs[cat][ARG_ERROR] = total_tcs[cat][ARG_ERROR] + 1
+          found = True
+        if ARG_MISMATCH in f:
+          tcs[ARG_MISMATCH] = tcs[ARG_MISMATCH] + 1
+          total_tcs[cat][ARG_MISMATCH] = total_tcs[cat][ARG_MISMATCH] + 1
+          found = True
+        if MISPLACED_CALL in f:
+          tcs[MISPLACED_CALL] = tcs[MISPLACED_CALL] + 1
+          total_tcs[cat][MISPLACED_CALL] = total_tcs[cat][MISPLACED_CALL] + 1
+          found = True
+        if MISSING_CALL in f:
+          tcs[MISSING_CALL] = tcs[MISSING_CALL] + 1
+          total_tcs[cat][MISSING_CALL] = total_tcs[cat][MISSING_CALL] + 1
+          found = True
+
+        if not found:
+          print('Did not find: ' + base_dir + '/conflo/' + cat + '/' + f)
+
+      print('=== Category: {cat} ===\n{the_dict}\n >> The erroneous state depends on the value of argv[1].'.format(cat=cat,the_dict=tcs))
+
+    # Code Duplication // FIXME
+    for cat in ['coll', 'datatype', 'pt2pt', 'rma']:
+      file_names = os.listdir(base_dir + '/correct/' + cat)
+      file_names.sort()
+      total_corrs = total_corrs + len(file_names)
+      print('=== Category: {cat} ===\nCorrect cases: {num_tcs}'.format(cat=cat,num_tcs=len(file_names)))
+
+    sum_str = ''
+    for cd in total_tcs:
+      sum_str = sum_str + '[' + cd + ']\t' + str(total_tcs[cd]) + '\n'
+      for k in total_tcs[cd]:
+        total_errs = total_errs + total_tcs[cd][k]
+    the_total = total_corrs + total_errs
+    print('\n\n>>> ==== SUMMARY of MPI CorrBench Test cases ==== <<<\n\n{summary}\n[NOTES]:\n{notes}\n\n===== >>> Total number of test cases: {total} | Errors/Correct: {errs}/{corrs} <<< ====\n'.format(summary=sum_str, total=the_total, notes=notes_str, errs=total_errs, corrs=total_corrs))
+
+
+    # Obtain the error descriptions // FIXME
+    file_names = os.listdir(dir_name)
+    file_names.sort()
+
+    print("Found " + str(len(file_names)) + " error case files in " + dir_name)
+
+    error_cases = {}
+
+    for file_name in all_files:
+        print('The current file ' + file_name)
+        error_cases[file_name] = {}
+        error_cases[file_name]["ITAC"] = ""
+        error_cases[file_name]["MPI-Checker"] = ""
+        error_cases[file_name]["MUST"] = ""
+
+        # read the file to discover error specification from comment
+        file_path = os.path.join(dir_name, file_name)
+        with open(file_name, "r") as file:
+            lines = file.readlines()
+            read_spec = False
+            error_specification = ""
+            for line in lines:
+                if not read_spec and "/*" in line:
+                    read_spec = True
+                elif read_spec:
+                    if "*/" in line:
+                        break
+                    error_specification = error_specification + line
+            error_specification = error_specification.replace("*", "")
+            error_specification = error_specification.replace("\n", "")
+            error_cases[file_name]["Error_Specification"] = error_specification.strip()
+
+    with open("error-cases.json", "w") as error_cases_file:
+        json.dump(error_cases, error_cases_file, indent=4)
+
+if __name__ == '__main__':
+        main()

micro-benches/0-level/coll/ArgError-MPIAllgather-Communicator-1.c

@@ -0,0 +1,28 @@
+#include <mpi.h>
+#include <stddef.h>
+#include <stdio.h>
+/*
+ * MPI communicator for Allgather is MPI_NULL_COMM. line 17
+ *
+ */
+int main(int argc, char *argv[]) {
+  int myRank, numProcs;
+
+  int local_sum = 4;
+  int global_sum[4];
+
+  MPI_Init(&argc, &argv);
+  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
+  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+
+  MPI_Comm null_comm = MPI_COMM_NULL;
+  MPI_Allgather(&local_sum, 1, MPI_INT, global_sum, 1, MPI_INT, null_comm);
+
+  if (myRank == 0) {
+    printf("Result: %d", global_sum);
+  }
+
+  MPI_Finalize();
+
+  return 0;
+}

micro-benches/0-level/coll/ArgError-MPIAllgather-Communicator-2.c

@@ -0,0 +1,28 @@
+#include <mpi.h>
+#include <stddef.h>
+#include <stdio.h>
+/*
+ * MPI communicator for Allgather is NULL. line 17
+ *
+ */
+int main(int argc, char *argv[]) {
+  int myRank, numProcs;
+
+  int local_sum = 4;
+  int global_sum[4];
+
+  MPI_Init(&argc, &argv);
+  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
+  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+
+  MPI_Comm null_comm = NULL;
+  MPI_Allgather(&local_sum, 1, MPI_INT, global_sum, 1, MPI_INT, null_comm);
+
+  if (myRank == 0) {
+    printf("Result: %d", global_sum[0]);
+  }
+
+  MPI_Finalize();
+
+  return 0;
+}

micro-benches/0-level/coll/ArgError-MPIAllgather-Count-1.c

@@ -0,0 +1,27 @@
+#include <mpi.h>
+#include <stddef.h>
+#include <stdio.h>
+/*
+ * Wrong number of elements to send in MPI_Allgather (does not match send buffer).. line 17
+ *
+ */
+int main(int argc, char *argv[]) {
+  int myRank, numProcs;
+
+  int local_sum = 4;
+  int global_sum[4];
+
+  MPI_Init(&argc, &argv);
+  MPI_Comm_size(MPI_COMM_WORLD, &numProcs);
+  MPI_Comm_rank(MPI_COMM_WORLD, &myRank);
+
+  MPI_Allgather(&local_sum, 2, MPI_INT, global_sum, 2, MPI_INT, MPI_COMM_WORLD);
+
+  if (myRank == 0) {
+    printf("Result: %d", global_sum);
+  }
+
+  MPI_Finalize();
+
+  return 0;
+}