Improve instrumentation support (#4747)

Fixes #4740, fixes #2509 Description of changes: - third-party tools: - fix the broken Caliper integration and write an integration test - use standard Caliper macros instead of macro aliases - build Caliper as a CMake subproject to inherit options (e.g. CUDA and MPI support) and facilitate linking - add native support for CUDA-GDB (a few compiler flags were missing from `Debug` builds) - add native support for kernprof (new flag added to `pypresso`) - documentation: - centralize scattered information about debuggers into a self-contained section of the user guide - document all supported debuggers and profilers: Caliper, Valgrind, GDB, CUDA-GDB, kernprof, perf, sanitizers - make the Doxygen docs more consistent with the Sphinx user guide (same fonts) and website (same logo)
espressomd · Jul 14, 2023 · 4c03ac4 · 4c03ac4
2 parents 3fd0a82 + a651397
commit 4c03ac4
Show file tree

Hide file tree

Showing 35 changed files with 1,320 additions and 328 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -68,7 +68,7 @@ style_doxygen:
     - mkdir build
     - cd build
     - cp ../maintainer/configs/maxset.hpp myconfig.hpp
-    - cmake .. -D ESPRESSO_BUILD_WITH_CUDA=ON -D ESPRESSO_BUILD_WITH_GSL=ON -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=ON -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_WALBERLA_FFT=ON -D ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS=ON
+    - cmake .. -D ESPRESSO_BUILD_WITH_CUDA=ON -D ESPRESSO_BUILD_WITH_GSL=ON -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=ON -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_WALBERLA_FFT=ON -D ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS=ON -D ESPRESSO_BUILD_WITH_CALIPER=ON
     - sh ../maintainer/CI/dox_warnings.sh
   tags:
     - espresso
@@ -111,6 +111,7 @@ maxset:
      with_scafacos: 'true'
      with_walberla: 'true'
      with_stokesian_dynamics: 'true'
+     with_caliper: 'true'
      check_skip_long: 'true'
      cmake_params: '-D ESPRESSO_TEST_NP=8'
   script:
@@ -241,6 +242,7 @@ cuda11-maxset:
      with_walberla: 'true'
      with_walberla_avx: 'true'
      with_stokesian_dynamics: 'true'
+     with_caliper: 'true'
   script:
     - bash maintainer/CI/build_cmake.sh
   artifacts:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -99,8 +99,8 @@ option(ESPRESSO_BUILD_WITH_WALBERLA_AVX
        "Build waLBerla lattice-Boltzmann with AVX vectorization" OFF)
 option(ESPRESSO_BUILD_WITH_WALBERLA_FFT "Build waLBerla with FFT support" OFF)
 option(ESPRESSO_BUILD_BENCHMARKS "Enable benchmarks" OFF)
-option(ESPRESSO_BUILD_WITH_VALGRIND_MARKERS
-       "Build with valgrind instrumentation markers" OFF)
+option(ESPRESSO_BUILD_WITH_VALGRIND "Build with Valgrind instrumentation" OFF)
+option(ESPRESSO_BUILD_WITH_CALIPER "Build with Caliper instrumentation" OFF)
 option(ESPRESSO_BUILD_WITH_CPPCHECK "Run Cppcheck during compilation" OFF)
 if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   option(ESPRESSO_BUILD_WITH_CLANG_TIDY "Run Clang-Tidy during compilation" OFF)
@@ -128,8 +128,6 @@ option(
 option(ESPRESSO_WARNINGS_ARE_ERRORS
        "Treat warnings as errors during compilation" OFF)
 option(ESPRESSO_BUILD_WITH_CCACHE "Use ccache compiler invocation." OFF)
-option(ESPRESSO_BUILD_WITH_PROFILER "Build with Caliper profiler annotations."
-       OFF)
 option(ESPRESSO_INSIDE_DOCKER "Set this to ON when running inside Docker." OFF)
 mark_as_advanced(ESPRESSO_INSIDE_DOCKER)
 set(ESPRESSO_TEST_TIMEOUT "300"
@@ -348,12 +346,11 @@ if(ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS)
   endif()
 endif()
 
-if(ESPRESSO_BUILD_WITH_VALGRIND_MARKERS)
+if(ESPRESSO_BUILD_WITH_VALGRIND)
   find_package(PkgConfig REQUIRED)
   pkg_check_modules(VALGRIND valgrind REQUIRED)
   if(VALGRIND_FOUND)
-    message(STATUS ${VALGRIND_INCLUDE_DIRS})
-    include_directories(SYSTEM ${VALGRIND_INCLUDE_DIRS})
+    message(STATUS "Found valgrind: ${VALGRIND_INCLUDE_DIRS}")
   endif()
 endif()
 
@@ -657,6 +654,33 @@ if(ESPRESSO_BUILD_WITH_WALBERLA)
   endif()
 endif()
 
+if(ESPRESSO_BUILD_WITH_CALIPER)
+  # cmake-format: off
+  include(FetchContent)
+  FetchContent_Declare(
+    caliper
+    GIT_REPOSITORY https://github.com/LLNL/Caliper.git
+    GIT_TAG        v2.9.1
+  )
+  if(NOT DEFINED caliper_SOURCE_DIR OR NOT EXISTS "${caliper_SOURCE_DIR}")
+    FetchContent_Populate(caliper)
+  endif()
+  # cmake-format: on
+  set(CALIPER_OPTION_PREFIX on CACHE BOOL "")
+  set(CALIPER_WITH_MPI on CACHE BOOL "")
+  if(ESPRESSO_BUILD_WITH_CUDA)
+    set(CALIPER_WITH_NVTX on CACHE BOOL "")
+    set(CALIPER_WITH_CUPTI on CACHE BOOL "")
+  endif()
+  set(CALIPER_BUILD_SHARED_LIBS on CACHE BOOL "")
+  add_subdirectory("${caliper_SOURCE_DIR}")
+  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL
+                                              "GNU")
+    target_compile_options(caliper-services
+                           PRIVATE -Wno-deprecated-declarations)
+  endif()
+endif()
+
 #
 # Subdirectories
 #

diff --git a/cmake/FindCUDACompilerNVCC.cmake b/cmake/FindCUDACompilerNVCC.cmake
@@ -43,7 +43,7 @@ add_library(espresso::cuda_flags ALIAS espresso_cuda_flags)
 target_compile_options(
   espresso_cuda_flags
   INTERFACE
-  $<$<CONFIG:Debug>:>
+  $<$<CONFIG:Debug>:-g -G>
   $<$<CONFIG:Release>:-Xptxas=-O3 -Xcompiler=-O3 -DNDEBUG>
   $<$<CONFIG:MinSizeRel>:-Xptxas=-O2 -Xcompiler=-Os -DNDEBUG>
   $<$<CONFIG:RelWithDebInfo>:-Xptxas=-O2 -Xcompiler=-O2,-g -DNDEBUG>

diff --git a/cmake/espresso_cmake_config.cmakein b/cmake/espresso_cmake_config.cmakein
@@ -17,7 +17,9 @@
 
 #cmakedefine ESPRESSO_BUILD_WITH_WALBERLA_FFT
 
-#cmakedefine ESPRESSO_BUILD_WITH_VALGRIND_MARKERS
+#cmakedefine ESPRESSO_BUILD_WITH_VALGRIND
+
+#cmakedefine ESPRESSO_BUILD_WITH_CALIPER
 
 #define PACKAGE_NAME "${PROJECT_NAME}"
 

diff --git a/doc/bibliography.bib b/doc/bibliography.bib
@@ -98,28 +98,30 @@ @Article{arnold05b
 }
 
 @InCollection{arnold13a,
-  author = {A. Arnold and O. Lenz and S. Kesselheim and R. Weeber and F. Fahrenberger and D. R\"{o}hm and P. Ko\v{s}ovan and C. Holm},
-  title = {{ESPResSo}~3.1 --- Molecular Dynamics Software for Coarse-Grained Models},
+  author    = {Arnold, Axel and Lenz, Olaf and Kesselheim, Stefan and Weeber, Rudolf and Fahrenberger, Florian and R\"{o}hm, Dominic and Ko\v{s}ovan, Peter and Holm, Christian},
+  title     = {{ESPResSo}~3.1 -- Molecular Dynamics Software for Coarse-Grained Models},
   booktitle = {Meshfree Methods for Partial Differential Equations {VI}},
+  year      = {2013},
+  editor    = {Griebel, Michael and Schweitzer, Marc Alexander},
+  volume    = {89},
+  series    = {Lecture Notes in Computational Science and Engineering},
   publisher = {Springer Berlin Heidelberg},
-  year = {2013},
-  editor = {M. Griebel and M. A. Schweitzer},
-  series = {Lecture Notes in Computational Science and Engineering},
-  pages = {1--23},
-  volume = {89},
-  doi = {10.1007/978-3-642-32979-1_1},
+  pages     = {1--23},
+  doi       = {10.1007/978-3-642-32979-1_1},
 }
 
 @Article{ballenegger09a,
-  author = {V. Ballenegger and A. Arnold and J. J. Cerda},
-  title = {Simulations of non-neutral slab systems with long-range electrostatic interactions in two-dimensional periodic boundary conditions},
-  journal = {The Journal of Chemical Physics},
-  year = {2009},
-  volume = {131},
-  pages = {094107},
-  number = {9},
-  doi = {10.1063/1.3216473},
-  numpages = {10},
+  author    = {Ballenegger, Vincent and Arnold, Axel and Cerd\`a, Joan J.},
+  title     = {Simulations of non-neutral slab systems with long-range electrostatic interactions in two-dimensional periodic boundary conditions},
+  journal   = {The Journal of Chemical Physics},
+  year      = {2009},
+  volume    = {131},
+  number    = {9},
+  month     = sep,
+  pages     = {094107},
+  issn      = {1089-7690},
+  doi       = {10.1063/1.3216473},
+  numpages  = {10},
   publisher = {AIP},
 }
 
@@ -191,6 +193,19 @@ @Article{bindgen21a
   pages = {083615},
 }
 
+@InProceedings{boehme16a,
+  author    = {Boehme, David and Gamblin, Todd and Beckingsale, David and Bremer, Peer-Timo and Gimenez, Alfredo and LeGendre, Matthew and Pearce, Olga and Schulz, Martin},
+  title     = {{C}aliper: Performance Introspection for {HPC} Software Stacks},
+  booktitle = {{SC} '16: Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis},
+  year      = {2016},
+  note      = {LLNL-CONF-699263},
+  publisher = {{IEEE} Press},
+  location  = {Salt Lake City, Utah, USA},
+  month     = nov,
+  isbn      = {978-1-4673-8815-3},
+  pages     = {550--560},
+}
+
 @Article{brady88a,
   author  = {Brady, John F. and Bossis, Georges},
   title   = {{S}tokesian Dynamics},
@@ -483,7 +498,7 @@ @Book{frenkel02b
   author     = {Frenkel, Daan and Smit, Berend},
   title      = {Understanding Molecular Simulation: {F}rom Algorithms to Applications},
   year       = {2002},
-  edition    = {2nd},
+  edition    = {2\textsuperscript{nd}},
   publisher  = {Academic Press},
   isbn       = {978-0-12-267351-1},
   doi        = {10.1016/B978-0-12-267351-1.X5000-7},
@@ -623,6 +638,19 @@ @Article{kesselheim11a
   issn = {0010-4655},
 }
 
+@InProceedings{kluyver16a,
+  author    = {Kluyver, Thomas and Ragan-Kelley, Benjamin and P{\'e}rez, Fernando and Granger, Brian E and Bussonnier, Matthias and Frederic, Jonathan and Kelley, Kyle and Hamrick, Jessica B and Grout, Jason and Corlay, Sylvain and Ivanov, Paul and Avila, Dami{\'a}n and Abdalla, Safia and Willing, Carol and {Jupyter Development Team}},
+  title     = {{J}upyter {N}otebooks---a publishing format for reproducible computational workflows},
+  booktitle = {Positioning and Power in Academic Publishing: Players, Agents and Agendas},
+  year      = {2016},
+  editor    = {Loizides, Fernando and Schmidt, Birgit},
+  publisher = {IOS Press},
+  location  = {Amsterdam},
+  isbn      = {978-1-61499-649-1},
+  pages     = {87--90},
+  doi       = {10.3233/978-1-61499-649-1-87},
+}
+
 @Article{kolafa92a,
   author = {Kolafa, Jiri and Perram, John W.},
   title = {Cutoff Errors in the {E}wald Summation Formulae for Point Charge Systems},
@@ -757,6 +785,38 @@ @Article{martys99a
   doi = {10.1103/PhysRevE.59.3733},
 }
 
+@TechReport{misc-cuda-gdb,
+  author      = {{NVIDIA}},
+  title       = {{CU-GDB}},
+  type        = {User manual},
+  number      = {Release 12.1},
+  year        = {2023},
+  month       = apr,
+  institution = {NVIDIA},
+  url         = {https://docs.nvidia.com/cuda/pdf/cuda-gdb.pdf},
+}
+
+@Misc{misc-kernprof,
+  author    = {Kern, Robert and Crall, Jon and Olsen, Brett and {OpenPyUtils community contributors}},
+  title     = {{line_profiler} and {kernprof}},
+  year      = {2008},
+  url       = {https://github.com/pyutils/line_profiler},
+}
+
+@Misc{misc-perf,
+  author    = {Torvalds, Linus},
+  title     = {{perf}: {L}inux profiling with performance counters},
+  year      = {2009},
+  url       = {https://perf.wiki.kernel.org/index.php/Main_Page},
+}
+
+@Misc{misc-ubsan,
+  author    = {{LLVM Developers}},
+  title     = {Undefined Behavior Sanitizer},
+  year      = {2017},
+  url       = {https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html},
+}
+
 @Book{moshier89a,
   title={Methods and Programs for Mathematical Functions},
   author={Moshier, Stephen Lloyd Baluk},
@@ -777,6 +837,33 @@ @Article{moss96a
   doi       = {10.1351/pac199668122193},
 }
 
+@Article{nethercote03a,
+  author    = {Nethercote, Nicholas and Seward, Julian},
+  title     = {{V}algrind: A Program Supervision Framework},
+  journal   = {Electronic Notes in Theoretical Computer Science},
+  year      = {2003},
+  volume    = {89},
+  number    = {2},
+  month     = oct,
+  pages     = {44--66},
+  issn      = {1571-0661},
+  doi       = {10.1016/S1571-0661(04)81042-9},
+}
+
+@InProceedings{nethercote07a,
+  author    = {Nethercote, Nicholas and Seward, Julian},
+  title     = {{V}algrind: A Framework for Heavyweight Dynamic Binary Instrumentation},
+  booktitle = {Proceedings of the 28\textsuperscript{th} {ACM} {SIGPLAN} Conference on Programming Language Design and Implementation},
+  year      = {2007},
+  publisher = {Association for Computing Machinery},
+  location  = {San Diego, California, USA},
+  isbn      = {978-1-59593-633-2},
+  month     = jun,
+  pages     = {89--100},
+  doi       = {10.1145/1250734.1250746},
+  address   = {New York, New York, USA},
+}
+
 @Article{neumann85b,
 author = {Neumann, Martin},
 title = {The dielectric constant of water. Computer simulations with the {MCY} potential},
@@ -880,7 +967,7 @@ @Book{rapaport04a
 
 @Article{reed92a,
   title={{M}onte {C}arlo study of titration of linear polyelectrolytes},
-  author={Reed, Christopher E and Reed, Wayne F},
+  author={Reed, Christopher E. and Reed, Wayne F.},
   journal={The Journal of Chemical Physics},
   volume={96},
   number={2},
@@ -910,15 +997,27 @@ @Article{schatzel88a
 }
 
 @Book{schlick10a,
-address = {New York, NY},
 author = {Schlick, Tamar},
-doi = {10.1007/978-1-4419-6351-2},
-isbn = {978-1-4419-6350-5},
-publisher = {Springer New York},
 series = {Interdisciplinary Applied Mathematics},
 title = {Molecular Modeling and Simulation: {A}n Interdisciplinary Guide},
 volume = {21},
 year = {2010},
+address = {New York, New York, USA},
+doi = {10.1007/978-1-4419-6351-2},
+isbn = {978-1-4419-6350-5},
+publisher = {Springer New York},
+}
+
+@InProceedings{serebryany12a,
+  author    = {Serebryany, Konstantin and Bruening, Derek and Potapenko, Alexander and Vyukov, Dmitry},
+  title     = {{A}ddress{S}anitizer: A Fast Address Sanity Checker},
+  booktitle = {{USENIX} Annual Technical Conference},
+  year      = {2012},
+  publisher = {USENIX Association},
+  location  = {Boston, Massachusetts, USA},
+  month     = jun,
+  url       = {https://www.usenix.org/conference/atc12/addresssanitizer-fast-address-sanity-checker},
+  address   = {Berkeley, California, USA},
 }
 
 @Article{smith81a,
@@ -975,8 +1074,17 @@ @Article{sonnenschein85a
 doi = {10.1016/0021-9991(85)90151-2},
 }
 
+@Book{stallman11a,
+  author    = {Stallman, Richard and Pesch, Roland and Shebs, Stan},
+  title     = {Debugging with {GDB}: The {GNU} source-level debugger},
+  year      = {2011},
+  edition   = {10th},
+  publisher = {{GNU} Press},
+  isbn      = {978-0-9831592-3-0},
+}
+
 @PhdThesis{strebel99a,
-  author = {Strebel, R.},
+  author = {Strebel, Rolf},
   title = {Pieces of software for the {C}oulombic m body problem},
   school = {ETH Z\"{u}rich},
   year = {1999},
@@ -1128,6 +1236,21 @@ @Article{weik19a
   doi       = {10.1140/epjst/e2019-800186-9},
 }
 
+@InProceedings{weidendorfer04a,
+  author    = {Weidendorfer, Josef and Kowarschik, Markus and Trinitis, Carsten},
+  title     = {A Tool Suite for Simulation Based Analysis of Memory Access Behavior},
+  booktitle = {Computational Science - ICCS 2004},
+  year      = {2004},
+  editor    = {Bubak, Marian and van Albada, Geert Dick and Sloot, Peter M. A. and Dongarra, Jack},
+  volume    = {3038},
+  series    = {Lecture Notes in Computer Science},
+  publisher = {Springer Berlin Heidelberg},
+  isbn      = {978-3-540-24688-6},
+  pages     = {440--447},
+  doi       = {10.1007/978-3-540-24688-6_58},
+  address   = {Berlin, Heidelberg},
+}
+
 @Article{widom63a,
   author    = {Widom, B.},
   title     = {Some topics in the theory of fluids},

diff --git a/doc/doxygen/CMakeLists.txt b/doc/doxygen/CMakeLists.txt
@@ -33,21 +33,22 @@ if(DOXYGEN_FOUND)
   add_custom_command(
     OUTPUT ${DOXYGEN_BIB_OUT}
     COMMAND
-      sed -r
-      "'s_^ *doi *= *([^0-9]+)(10\\.[0-9]+)_url=\\1https://dx.doi.org/\\2_'"
+      sed -r "'s_^ *doi *= *([^0-9]+)(10\\.[0-9]+)_url=\\1https://doi.org/\\2_'"
       ${DOXYGEN_BIB_IN} > ${DOXYGEN_BIB_OUT}
     DEPENDS ${DOXYGEN_BIB_IN})
 
-  set(DOXYGEN_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in)
-  set(DOXYGEN_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+  set(DOXYFILE_IN ${CMAKE_CURRENT_SOURCE_DIR}/Doxyfile.in)
+  set(DOXYFILE_OUT ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile)
+  set(CITELIST ${CMAKE_CURRENT_BINARY_DIR}/html/citelist.html)
 
-  configure_file(${DOXYGEN_IN} ${DOXYGEN_OUT} @ONLY)
+  configure_file(${DOXYFILE_IN} ${DOXYFILE_OUT} @ONLY)
 
   add_custom_target(
-    doxygen COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYGEN_OUT}
+    doxygen COMMAND ${DOXYGEN_EXECUTABLE} ${DOXYFILE_OUT}
+    COMMAND sed -ri "s/ textsuperscript (\\S+) /<sup>\\1<\\/sup> /" ${CITELIST}
+    COMMAND sed -ri "s/ textsubscript (\\S+) /<sub>\\1<\\/sub> /" ${CITELIST}
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     COMMENT "Generating API documentation with Doxygen"
     DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/doxy-features ${DOXYGEN_BIB_OUT}
     VERBATIM)
-
 endif(DOXYGEN_FOUND)