diff --git a/CMakeLists.txt b/CMakeLists.txt index c359bdd7c58..ac155e74796 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,16 +62,6 @@ if(MSVC) endif() option(OGS_USE_MKL "Use Intel MKL" OFF) -if(OGS_USE_MKL) - option(MKL_USE_parallel "Use MKL parallel" True) - option(MKL_USE_sdl "Single Dynamic Library or static/dynamic" False) - set(MKL_USE_interface - "lp64" - CACHE - STRING - "for Intel(R)64 compatible arch: ilp64/lp64 or for ia32 arch: cdecl/stdcall" - ) -endif() # Eigen option(OGS_USE_EIGEN_UNSUPPORTED "Use Eigen unsupported modules" ON) diff --git a/MathLib/CMakeLists.txt b/MathLib/CMakeLists.txt index 03b2597e597..fbf64644403 100644 --- a/MathLib/CMakeLists.txt +++ b/MathLib/CMakeLists.txt @@ -32,6 +32,7 @@ target_link_libraries( $<$:${LIS_LIBRARIES}> $<$:CVODE::CVODE> $<$:petsc> + $<$:MKL::MKL> Eigen3::Eigen $<$:OpenMP::OpenMP_CXX> ) @@ -39,13 +40,6 @@ target_link_libraries( if(OGS_USE_LIS) target_include_directories(MathLib PUBLIC ${LIS_INCLUDE_DIR}) endif() -if(OGS_USE_MKL) - target_include_directories(MathLib PUBLIC ${MKL_INCLUDE_DIR}) - target_link_libraries(MathLib PUBLIC ${MKL_LIBRARIES}) - target_link_options(MathLib PUBLIC ${MKL_LINK_FLAGS}) - target_compile_definitions(MathLib PUBLIC ${MKL_DEFINITIONS}) - target_link_directories(MathLib PUBLIC ${MKL_LIBRARY_DIR}) -endif() target_compile_definitions( MathLib diff --git a/scripts/ci/extends/vs2019-environment.yml b/scripts/ci/extends/vs2019-environment.yml index f035b276ef0..fdcda29d0e7 100644 --- a/scripts/ci/extends/vs2019-environment.yml +++ b/scripts/ci/extends/vs2019-environment.yml @@ -1,13 +1,22 @@ +.load_vs2019_env: &load_vs2019_env + - | + pushd $env:VS160COMNTOOLS + cmd /c "VsDevCmd.bat -arch=amd64 -host_arch=amd64&set" | + foreach { + if ($_ -match "=") { + $v = $_.split("=", 2); set-item -force -path "ENV:\$($v[0])" -value "$($v[1])" + } + } + popd + Write-Host "`nVisual Studio Command Prompt variables set." -ForegroundColor Yellow + .vs2019-environment: before_script: - # Load VS environment - - | - pushd $env:VS160COMNTOOLS - cmd /c "VsDevCmd.bat -arch=amd64 -host_arch=amd64&set" | - foreach { - if ($_ -match "=") { - $v = $_.split("=", 2); set-item -force -path "ENV:\$($v[0])" -value "$($v[1])" - } - } - popd - Write-Host "`nVisual Studio Command Prompt variables set." -ForegroundColor Yellow + - *load_vs2019_env + +.vs2019-mkl-environment: + before_script: + - *load_vs2019_env + # Load MKL environment + - '& "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"' + - $env:PATH += ";C:\Program Files (x86)\Intel\oneAPI\compiler\latest\windows\redist\intel64_win\compiler" diff --git a/scripts/ci/jobs/build-linux-arch.yml b/scripts/ci/jobs/build-linux-arch.yml index 867845b3355..106fbc8a384 100644 --- a/scripts/ci/jobs/build-linux-arch.yml +++ b/scripts/ci/jobs/build-linux-arch.yml @@ -11,3 +11,5 @@ build linux arch: CMAKE_ARGS: >- -DBUILD_SHARED_LIBS=ON -DOGS_USE_MKL=ON + before_script: + - source /opt/intel/oneapi/setvars.sh diff --git a/scripts/ci/jobs/build-win.yml b/scripts/ci/jobs/build-win.yml index 85fb64c453a..c4e687de1c6 100644 --- a/scripts/ci/jobs/build-win.yml +++ b/scripts/ci/jobs/build-win.yml @@ -1,7 +1,7 @@ build win: extends: - .template-build-win - - .vs2019-environment + - .vs2019-mkl-environment variables: CHECK_WARNINGS: "true" CMAKE_PRESET: release diff --git a/scripts/cmake/Find.cmake b/scripts/cmake/Find.cmake index 0bff3d8fc9d..f692ad304fc 100644 --- a/scripts/cmake/Find.cmake +++ b/scripts/cmake/Find.cmake @@ -82,23 +82,42 @@ if(NOT (OGS_USE_PETSC AND OGS_USE_MKL) OR OGS_USE_PETSC_MKL_EIGEN_OPENMP) find_package(OpenMP COMPONENTS C CXX) endif() -# blas / lapack +# blas / lapack / MKL if(OGS_USE_MKL) - if("${MKL_USE_interface}" STREQUAL "lp64") + if(APPLE) + set(_mac_ld_prefix "DY") + endif() + if(NOT DEFINED ENV{MKLROOT} OR (NOT "$ENV{${_mac_ld_prefix}LD_LIBRARY_PATH}" + MATCHES "intel" AND NOT WIN32) + ) + message( + FATAL_ERROR + "OGS_USE_MKL was used but it seems that you did not source the MKL environment. " + "Typically you can run `source /opt/intel/oneapi/setvars.sh` before running CMake." + ) + endif() + set(MKL_INTERFACE + "lp64" + CACHE + STRING + "for Intel(R)64 compatible arch: ilp64/lp64 or for ia32 arch: cdecl/stdcall" + ) + if("${MKL_INTERFACE}" STREQUAL "lp64") set(BLA_VENDOR Intel10_64lp) - elseif("${MKL_USE_interface}" STREQUAL "ilp64") + elseif("${MKL_INTERFACE}" STREQUAL "ilp64") set(BLA_VENDOR Intel10_64ilp) endif() + if(NOT WIN32 AND NOT APPLE) + set(CMAKE_REQUIRE_FIND_PACKAGE_BLAS TRUE) + set(CMAKE_REQUIRE_FIND_PACKAGE_LAPACK TRUE) + endif() endif() find_package(BLAS) find_package(LAPACK) if(OGS_USE_MKL) - find_package(MKL REQUIRED) - find_file(MKL_SETVARS setvars.sh PATHS ${MKL_ROOT_DIR} ${MKL_ROOT_DIR}/.. - ${MKL_ROOT_DIR}/../.. - NO_DEFAULT_PATH - ) + find_package(MKL CONFIG REQUIRED PATHS $ENV{MKLROOT}) + find_file(MKL_SETVARS setvars.sh PATHS ${MKL_ROOT}/../.. NO_DEFAULT_PATH) endif() # Check MPI package @@ -134,7 +153,7 @@ function(printMKLUsage) if(WIN32) message( STATUS - "NOTE: Please add the MKL redist directory to your PATH environment variable!\nE.g. with: set PATH=%PATH%;${MKL_ROOT_DIR}/redist/intel64" + "NOTE: In addition to loading the oneAPI setvars.bat file please also add this to your PATH C:\\Program Files (x86)\\Intel\\oneAPI\\compiler\\latest\\windows\\redist\\intel64_win\\compiler" ) else() message( diff --git a/scripts/cmake/FindMKL.cmake b/scripts/cmake/FindMKL.cmake deleted file mode 100644 index edc70d7f3ac..00000000000 --- a/scripts/cmake/FindMKL.cmake +++ /dev/null @@ -1,354 +0,0 @@ -# ------------------------------------------------------------------- -# Originally obtained from https://github.com/openmeeg/findmkl_cmake. -# Modified for OpenGeoSys in https://github.com/bilke/findmkl_cmake. -# ------------------------------------------------------------------- - -# Distributed under the OSI-approved BSD 3-Clause License. See accompanying -# file Copyright.txt or https://cmake.org/licensing for details. - -# .rst: FindMKL -# ------- -# -# Find a Intel® Math Kernel Library (Intel® MKL) installation and provide all -# necessary variables and macros to compile software for it. -# -# MKLROOT is required in your system -# -# we use mkl_link_tool to get the library needed depending on variables There -# are few sets of libraries: -# -# Array indexes modes: -# -# :: -# -# LP - 32 bit indexes of arrays ILP - 64 bit indexes of arrays -# -# Threading: -# -# :: -# -# SEQUENTIAL - no threading INTEL - Intel threading library GNU - GNU threading -# library MPI support NOMPI - no MPI support INTEL - Intel MPI library OPEN - -# Open MPI library SGI - SGI MPT Library -# -# The following are set after the configuration is done: -# -# :: -# -# MKL_FOUND - system has MKL MKL_ROOT_DIR - path to the MKL base -# directory MKL_INCLUDE_DIR - the MKL include directory MKL_LIBRARIES - MKL -# libraries MKL_LIBRARY_DIR - MKL library dir (for dlls!) -# -# Sample usage: -# -# If MKL is required (i.e., not an optional part): -# -# :: -# -# find_package(MKL REQUIRED) if (MKL_FOUND) -# include_directories(${MKL_INCLUDE_DIR}) # and for each of your dependent -# executable/library targets: target_link_libraries( -# ${MKL_LIBRARIES}) endif() - -# NOTES -# -# If you want to use the module and your build type is not supported -# out-of-the-box, please contact me to exchange information on how your system -# is setup and I'll try to add support for it. -# -# AUTHOR -# -# Joan MASSICH (joan.massich-vall.AT.inria.fr). Alexandre GRAMFORT -# (Alexandre.Gramfort.AT.inria.fr) Théodore PAPADOPOULO (papadop.AT.inria.fr) - -set(FINDMKL_DEBUG_MODE 1) - -# unset this variable defined in matio -unset(MSVC) - -# Find MKL ROOT -find_path(MKL_ROOT_DIR NAMES include/mkl_cblas.h PATHS $ENV{MKLROOT} ${MKLROOT} - $ENV{MKL_DIR} ${MKL_DIR} -) - -# Convert symlinks to real paths - -get_filename_component(MKL_ROOT_DIR ${MKL_ROOT_DIR} REALPATH) - -if(NOT MKL_ROOT_DIR) - if(MKL_FIND_REQUIRED) - message( - FATAL_ERROR - "Could not find MKL: please set environment or CMake variable MKLROOT or MKL_DIR!" - ) - else() - unset(MKL_ROOT_DIR CACHE) - endif() -else() - # set arguments to call the MKL-provided tool for linking - set(MKL_INCLUDE_DIR ${MKL_ROOT_DIR}/include) - - # determine the name of the MKL-provided tool for linking - if(WIN32) - set(MKL_LINK_TOOL_BINARY mkl_link_tool.exe) - else() - set(MKL_LINK_TOOL_BINARY mkl_link_tool) - endif() - - # determine the path of the MKL-provided tool for linking - if(EXISTS "${MKL_ROOT_DIR}/bin/intel64/${MKL_LINK_TOOL_BINARY}") - # this path is used by MKL 2021.1.1 and newer - set(MKL_LINK_TOOL "${MKL_ROOT_DIR}/bin/intel64/${MKL_LINK_TOOL_BINARY}") - elseif(EXISTS "${MKL_ROOT_DIR}/bin/${MKL_LINK_TOOL_BINARY}") - # this path is used by MKL 2020.1.217 and newer - set(MKL_LINK_TOOL "${MKL_ROOT_DIR}/bin/${MKL_LINK_TOOL_BINARY}") - elseif(EXISTS "${MKL_ROOT_DIR}/tools/${MKL_LINK_TOOL_BINARY}") - set(MKL_LINK_TOOL "${MKL_ROOT_DIR}/tools/${MKL_LINK_TOOL_BINARY}") - else() - message( - FATAL_ERROR - "cannot find MKL link tool in any of the searched locations" - ) - endif() - - # now the libs - set(MKL_LINK_TOOL_COMMAND ${MKL_LINK_TOOL} "-libs") - - # possible versions - # <11.3|11.2|11.1|11.0|10.3|10.2|10.1|10.0|ParallelStudioXE2016|ParallelStudioXE2015|ComposerXE2013SP1|ComposerXE2013|ComposerXE2011|CompilerPro> - - # not older than MKL 10 (2011) - if(MKL_INCLUDE_DIR MATCHES "Composer.*2013") - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=ComposerXE2013") - elseif(MKL_INCLUDE_DIR MATCHES "Composer.*2011") - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=ComposerXE2011") - elseif(MKL_INCLUDE_DIR MATCHES "10.3") - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=10.3") - elseif(MKL_INCLUDE_DIR MATCHES "2013") # version 11 ... - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=11.1") - elseif(MKL_INCLUDE_DIR MATCHES "2015") - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=11.2") - elseif(MKL_INCLUDE_DIR MATCHES "2016") - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=11.3") - elseif(MKL_INCLUDE_DIR MATCHES "2017") - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=11.3") - elseif(MKL_INCLUDE_DIR MATCHES "2018") - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=11.3") - elseif(MKL_INCLUDE_DIR MATCHES "10") - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=10.2") - else() - list(APPEND MKL_LINK_TOOL_COMMAND "--mkl=11.3") - endif() - - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - list(APPEND MKL_LINK_TOOL_COMMAND "--compiler=clang") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - list(APPEND MKL_LINK_TOOL_COMMAND "--compiler=intel_c") - elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") - list(APPEND MKL_LINK_TOOL_COMMAND "--compiler=ms_c") - else() - list(APPEND MKL_LINK_TOOL_COMMAND "--compiler=gnu_c") - endif() - - if(APPLE) - list(APPEND MKL_LINK_TOOL_COMMAND "--os=mac") - elseif(WIN32) - list(APPEND MKL_LINK_TOOL_COMMAND "--os=win") - else() - list(APPEND MKL_LINK_TOOL_COMMAND "--os=lnx") - endif() - - set(MKL_LIB_DIR) - if(${CMAKE_SIZEOF_VOID_P} EQUAL 8) - list(APPEND MKL_LINK_TOOL_COMMAND "--arch=intel64") - set(MKL_LIB_DIR "intel64") - else() - list(APPEND MKL_LINK_TOOL_COMMAND "--arch=ia32") - set(MKL_LIB_DIR "ia32") - endif() - - if(MKL_USE_sdl) - list(APPEND MKL_LINK_TOOL_COMMAND "--linking=sdl") - else() - if(BLA_STATIC) - list(APPEND MKL_LINK_TOOL_COMMAND "--linking=static") - else() - list(APPEND MKL_LINK_TOOL_COMMAND "--linking=dynamic") - endif() - endif() - - if(MKL_USE_parallel) - list(APPEND MKL_LINK_TOOL_COMMAND "--parallel=yes") - else() - list(APPEND MKL_LINK_TOOL_COMMAND "--parallel=no") - endif() - - if(FORCE_BUILD_32BITS) - list(APPEND MKL_LINK_TOOL_COMMAND "--interface=cdecl") - set(MKL_USE_interface "cdecl" - CACHE STRING "disabled by FORCE_BUILD_32BITS" FORCE - ) - else() - list(APPEND MKL_LINK_TOOL_COMMAND "--interface=${MKL_USE_interface}") - endif() - - if(MKL_USE_parallel) - # With icc on linux, supported values by mkl_link_tool for threading - # library are: iomp5|tbb - if(WIN32) - list(APPEND MKL_LINK_TOOL_COMMAND "--threading-library=tbb") - else() - list(APPEND MKL_LINK_TOOL_COMMAND "--threading-library=iomp5") - list(APPEND MKL_LINK_TOOL_COMMAND "--openmp=iomp5") - endif() - endif() - - execute_process( - COMMAND ${MKL_LINK_TOOL_COMMAND} - OUTPUT_VARIABLE MKL_LIBS - RESULT_VARIABLE COMMAND_WORKED - TIMEOUT 2 - ERROR_QUIET - ) - - set(MKL_LIBRARIES) - set(MKL_LINK_FLAGS "") - - if(NOT ${COMMAND_WORKED} EQUAL 0) - message( - FATAL_ERROR - "Cannot find the MKL libraries correctly. Please check your MKL input variables and mkl_link_tool. The command executed was:\n ${MKL_LINK_TOOL_COMMAND}." - ) - endif() - - set(MKL_LIBRARY_DIR) - - if(WIN32) - set(MKL_LIBRARY_DIR - "${MKL_ROOT_DIR}/lib/${MKL_LIB_DIR}/" - "${MKL_ROOT_DIR}/../../tbb/latest/lib/intel64/vc_mt" - ) - - # remove unwanted break - string(REGEX REPLACE "\n" "" MKL_LIBS ${MKL_LIBS}) - - # get the list of libs - separate_arguments(MKL_LIBS) - foreach(i ${MKL_LIBS}) - find_library(FULLPATH_LIB ${i} PATHS "${MKL_LIBRARY_DIR}") - - if(FULLPATH_LIB) - list(APPEND MKL_LIBRARIES ${FULLPATH_LIB}) - elseif(i) - list(APPEND MKL_LIBRARIES ${i}) - endif() - unset(FULLPATH_LIB CACHE) - endforeach() - - else() # UNIX and macOS - # remove unwanted break - string(REGEX REPLACE "\n" "" MKL_LIBS ${MKL_LIBS}) - if(MKL_LINK_TOOL_COMMAND MATCHES "static") - string(REPLACE "$(MKLROOT)" "${MKL_ROOT_DIR}" MKL_LIBRARIES - ${MKL_LIBS} - ) - # hack for lin with libiomp5.a - if(APPLE) - string(REPLACE "-liomp5" - "${MKL_ROOT_DIR}/../compiler/lib/libiomp5.a" - MKL_LIBRARIES ${MKL_LIBRARIES} - ) - else() - string( - REPLACE - "-liomp5" - "${MKL_ROOT_DIR}/../compiler/lib/${MKL_LIB_DIR}/libiomp5.a" - MKL_LIBRARIES - ${MKL_LIBRARIES} - ) - endif() - separate_arguments(MKL_LIBRARIES) - else() # dynamic or sdl - # get the lib dirs - string(REGEX REPLACE "^.*-L[^/]+([^\ ]+).*" "${MKL_ROOT_DIR}\\1" - INTEL_LIB_DIR ${MKL_LIBS} - ) - if(NOT EXISTS ${INTEL_LIB_DIR}) - # Work around a bug in mkl 2018 - set(INTEL_LIB_DIR1 "${INTEL_LIB_DIR}_lin") - if(NOT EXISTS ${INTEL_LIB_DIR1}) - message( - FATAL_ERROR - "MKL installation broken. Directory ${INTEL_LIB_DIR} does not exist." - ) - endif() - set(INTEL_LIB_DIR ${INTEL_LIB_DIR1}) - endif() - set(MKL_LIBRARY_DIR ${INTEL_LIB_DIR}) - if(EXISTS - ${MKL_ROOT_DIR}/../../compiler/latest/linux/compiler/lib/${MKL_LIB_DIR} - ) - list( - APPEND - MKL_LIBRARY_DIR - ${MKL_ROOT_DIR}/../../compiler/latest/linux/compiler/lib/${MKL_LIB_DIR} - ) - endif() - - # get the list of libs - separate_arguments(MKL_LIBS) - - # set full path to libs - foreach(i ${MKL_LIBS}) - string(REGEX REPLACE " -" "-" i_mod ${i}) - string(REGEX REPLACE "-l([^\ ]+)" "\\1" i_mod ${i_mod}) - string(REGEX REPLACE "-L.*" "" i_mod ${i_mod}) - - find_library(FULLPATH_LIB ${i_mod} PATHS "${MKL_LIBRARY_DIR}") - - if(FULLPATH_LIB) - list(APPEND MKL_LIBRARIES ${FULLPATH_LIB}) - elseif(i) - if("${i}" MATCHES "^-[l|L].*") - string(REPLACE "$(MKLROOT)" "$ENV{MKLROOT}" i ${i}) - list(APPEND MKL_LIBRARIES ${i}) - else() - list(APPEND MKL_LINK_FLAGS ${i}) - endif() - endif() - unset(FULLPATH_LIB CACHE) - endforeach() - endif() - endif() - - # now definitions - string(REPLACE "-libs" "-opts" MKL_LINK_TOOL_COMMAND - "${MKL_LINK_TOOL_COMMAND}" - ) - execute_process( - COMMAND ${MKL_LINK_TOOL_COMMAND} - OUTPUT_VARIABLE RESULT_OPTS - TIMEOUT 2 - ERROR_QUIET - ) - string(REGEX MATCHALL "[-/]D[^\ ]*" MKL_DEFINITIONS ${RESULT_OPTS}) - - if(FINDMKL_DEBUG_MODE) - message(STATUS "Executed command: \n${MKL_LINK_TOOL_COMMAND}") - message(STATUS "Found MKL_LIBRARIES:\n${MKL_LIBRARIES} ") - message(STATUS "Found MKL_DEFINITIONS:\n${MKL_DEFINITIONS} ") - message(STATUS "Found MKL_LIBRARY_DIR:\n${MKL_LIBRARY_DIR} ") - message(STATUS "Found MKL_INCLUDE_DIR:\n${MKL_INCLUDE_DIR} ") - message(STATUS "Found MKL_LINK_FLAGS:\n${MKL_LINK_FLAGS} ") - endif() - - include(FindPackageHandleStandardArgs) - find_package_handle_standard_args( - MKL DEFAULT_MSG MKL_INCLUDE_DIR MKL_LIBRARIES - ) - - mark_as_advanced( - MKL_INCLUDE_DIR MKL_LIBRARIES MKL_DEFINITIONS MKL_ROOT_DIR - MKL_LINK_FLAGS - ) -endif() diff --git a/web/content/docs/devguide/advanced/configuration-options/index.md b/web/content/docs/devguide/advanced/configuration-options/index.md index ea7b62cb994..581afed3c49 100644 --- a/web/content/docs/devguide/advanced/configuration-options/index.md +++ b/web/content/docs/devguide/advanced/configuration-options/index.md @@ -46,7 +46,10 @@ CMake switches to enable / disable parts of OGS. - `OGS_CPU_ARCHITECTURE` - Optimizes for the given CPU architecture see [-march](https://gcc.gnu.org/onlinedocs/gcc-4.5.3/gcc/i386-and-x86_002d64-Options.html)-flag. Defaults to `native`. For redistributable binaries set to `generic` on Linux and `core2` on Mac OS. Can be disabled when set to `OFF`. - `CMAKE_LIBRARY_SEARCH_PATH` - Additional library installation path, e.g. `/opt/local` or `C:/libs` - `OGS_DEPENDENCY_VERSIONS` - Overwrite individual entries in `web/data/versions.json`. Should be quoted and `;`-separated, e.g.: `-DOGS_DEPENDENCY_VERSIONS="minimum_version.petsc=3.16.2;ctest.large_runtime=120"`. -- `OGS_USE_MKL` - Enables MKL support. Requires MKL to be installed on the system.[^MKL] A CMake or environment variable `MKLROOT` or `MKL_DIR` has to point to the MKL installation directory. To enable 64-bit array indices in MKL add `-DMKL_USE_interface=ilp64` on the first CMake run (with a clean CMake cache) but this seems [not supported by Eigen](https://libeigen.gitlab.io/docs/TopicUsingIntelMKL.html). -- `OGS_USE_PETSC_MKL_EIGEN_OPENMP` - Also enables OpenMP multithreading when PETSc and MKL is used. This pulls in another OpenMP implementation besides the Intel MKL OpenMP. **Experimental feature!** +- `OGS_USE_MKL` - Enables MKL support. Requires MKL (as part of the Intel oneAPI toolkit) to be [installed](https://www.intel.com/content/www/us/en/developer/articles/guide/installation-guide-for-oneapi-toolkits.html) on the system. + + Before configuring with CMake you also need to source the `setvars.sh`-script. On a typical installation run this: `source /opt/intel/oneapi/setvars.sh` (Windows: `& "C:\Program Files (x86)\Intel\oneAPI\setvars.bat"`). This will set the `MKLROOT` environment variable and also the `DYLD_LIBRARY_PATH` to contain the MKL library locations (both variables will be checked by CMake). -[^MKL]: The `mkl_link_tool` (part of the MKL installation) is used for setting up necessary compiler flags. It is a 32-bit executable. On Arch Linux you need to enable the [multilib-repository](https://wiki.archlinux.org/title/official_repositories#Enabling_multilib) and then install the package `lib32-gcc-libs` (otherwise you may get this error when trying to run it: `mkl_link_tool: No such file or directory`) On other Linux distributions you have to install the package `lib32stdc++6` (Ubuntu / Debian) or `libstdc++.i686` (yum-based)! On Windows please also install the *Intel oneAPI Threading Building Blocks* during installation! + To enable 64-bit array indices in MKL add `-DMKL_INTERFACE=ilp64` on the first CMake run (with a clean CMake cache) but this seems [not supported by Eigen](https://libeigen.gitlab.io/docs/TopicUsingIntelMKL.html). + +- `OGS_USE_PETSC_MKL_EIGEN_OPENMP` - Also enables OpenMP multithreading when PETSc and MKL is used. This pulls in another OpenMP implementation besides the Intel MKL OpenMP. **Experimental feature!**